From e98090c2b168835e4907a017483f5059a12f96f9 Mon Sep 17 00:00:00 2001 From: fasseg Date: Mon, 2 Dec 2013 16:16:00 +0100 Subject: [PATCH] Complete rewrite with: * automatically detect which version of fcrepo we are connecting to * simple random byte array template to decrease the RNG usage * per thread and overall throughput calculation * updated README and added duration log output * updated logging framework * factored out initial object creation and final purging --- README.md | 51 +-- pom.xml | 32 +- .../java/org/fcrepo/bench/ActionWorker.java | 75 +++++ src/main/java/org/fcrepo/bench/BenchTool.java | 209 ++++++++++++ .../org/fcrepo/bench/BenchToolEntity.java | 28 ++ .../java/org/fcrepo/bench/BenchToolFC3.java | 274 --------------- .../java/org/fcrepo/bench/BenchToolFC4.java | 311 ------------------ .../fcrepo/bench/BenchToolInputStream.java | 56 ++-- .../org/fcrepo/bench/BenchToolResult.java | 46 +++ .../org/fcrepo/bench/FCRepoBenchRunner.java | 197 +++++++++++ .../org/fcrepo/bench/Fedora3RestClient.java | 131 ++++++++ .../org/fcrepo/bench/Fedora4RestClient.java | 122 +++++++ .../org/fcrepo/bench/FedoraRestClient.java | 80 +++++ src/main/resources/logback.xml | 17 + 14 files changed, 995 insertions(+), 634 deletions(-) create mode 100644 src/main/java/org/fcrepo/bench/ActionWorker.java create mode 100644 src/main/java/org/fcrepo/bench/BenchTool.java create mode 100644 src/main/java/org/fcrepo/bench/BenchToolEntity.java delete mode 100644 src/main/java/org/fcrepo/bench/BenchToolFC3.java delete mode 100644 src/main/java/org/fcrepo/bench/BenchToolFC4.java create mode 100644 src/main/java/org/fcrepo/bench/BenchToolResult.java create mode 100644 src/main/java/org/fcrepo/bench/FCRepoBenchRunner.java create mode 100644 src/main/java/org/fcrepo/bench/Fedora3RestClient.java create mode 100644 src/main/java/org/fcrepo/bench/Fedora4RestClient.java create mode 100644 src/main/java/org/fcrepo/bench/FedoraRestClient.java create mode 100644 src/main/resources/logback.xml diff --git a/README.md b/README.md index 89d580d..a6c08e7 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,69 @@ BenchTool for Fedora 3/4 ======================================================================================= -This is a simple ingest benchmark to compare performance of Fedora 3 and Fedora 4. -The suite consists of two main classes responsible for running the benchmarks. +This is a simple ingest/retrieve/update/delete benchmark to compare performance of Fedora 3 and Fedora 4. +The suite consists of a jar file which can be run directly. The Fedora version will be automatically discovered depending on the given URL. +The url should include the context path of the webapp. For example `http://localhost:8080/fcrepo` for a Fedora version deployed at the context path `fcrepo`. -Fedora 3 --------- -Tool to run an ingest benchmark against Fedora 3 ### Usage ``` -BenchToolFC3 [ingest|read|update|delete] +usage: BenchTool + -a,--action The action to perform. Can be one of + ingest, read, update or delete. + [default=ingest] + -f,--fedora-url The URL of the Fedora instance. The url + must include the context path of the + webapp. [default=http://localhost:8080] + -h,--help print the help screen + -l,--log The log file to which the durations will + get written. [default=durations.log] + -n,--num-actions The number of actions performed. + [default=1] + -p,--password The user's password + -s,--size The size of the individual binaries + used. [default=1024] + -t,--num-threads The number of threads used for + performing all actions. [default=1] + -u,--user The fedora user name ``` +Fedora 3 +-------- + ##### Example Login with the user `fedoraAdmin` and the passwd `changeme` and ingest 1000 Objects each with one datastream of 1024kb size - ``` -#> java -cp bench-tool-${VERSION}-jar-with-dependencies.jar org.fcrepo.bench.BenchToolFC3 http://localhost:8080/fedora fedoraAdmin changeme 1000 1024 ingest +#> java -jar target/bench-tool-${VERSION}-jar-with-dependencies.jar -f http://localhost:8080/fedora -u fedoraAdmin -p changeme -s 1048576 -n 1000 ``` Login with the user `fedoraAdmin` and the passwd `changeme` and update 1000 Objects each with one datastream of 1024kb size ``` -#> java -cp bench-tool-${VERSION}-jar-with-dependencies.jar org.fcrepo.bench.BenchToolFC3 http://localhost:8080/fedora fedoraAdmin changeme 1000 1024 update +#> java -jar target/bench-tool-${VERSION}-jar-with-dependencies.jar -f http://localhost:8080/fedora -u fedoraAdmin -p changeme -s 1048576 -n 1000 -a update ``` Fedora 4 -------- -Tool to run an ingest benchmark against Fedora 4 - -### Usage - -``` -BenchToolFC4 [ingest|read|update|delete] -``` #### Example Ingest 1000 Objects each with one datastream of 1024kb size using a max of 15 threads ``` -#> java -cp bench-tool-${VERSION}-jar-with-dependencies.jar org.fcrepo.bench.BenchToolFC4 http://localhost:8080/fcrepo 1000 1024 15 +#> java -jar target/bench-tool-${VERSION}-jar-with-dependencies.jar -f http://localhost:8080/fcrepo -s 1048576 -n 1000 -t 15 ``` Delete 1000 Objects with a single thread ``` -#> java -cp bench-tool-${VERSION}-jar-with-dependencies.jar org.fcrepo.bench.BenchToolFC4 http://localhost:8080/fcrepo 1000 1024 1 delete +#> java -jar target/bench-tool-${VERSION}-jar-with-dependencies.jar -f http://localhost:8080/fcrepo -s 1048576 -n 1000 -t 15 -a delete ``` Results ------- -Both main classes will generate a file called `ingest.log` containing the request durations in milliseconds which can be turned easily into a graph by using e.g. gnuplot +The durations file can be easily turned into a graph using gnuplot +#### Example ``` -gnuplot> plot "ingest.log" title "FCRepo3 Ingest" with lines +gnuplot> plot "durations.log" title "Duration" with lines ``` diff --git a/pom.xml b/pom.xml index 51f1d98..d999ffa 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ org.apache.httpcomponents httpclient - 4.2.1 + 4.3.1 commons-io @@ -25,6 +25,16 @@ com.hp.hpl.jena jena 2.6.4 + + + log4j + log4j + + + slf4j-log4j12 + org.slf4j + + org.uncommons.maths @@ -41,6 +51,21 @@ + + commons-cli + commons-cli + 1.2 + + + ch.qos.logback + logback-classic + 0.9.16 + + + ch.qos.logback + logback-core + 0.9.16 + @@ -56,6 +81,11 @@ maven-assembly-plugin + + + org.fcrepo.bench.BenchTool + + jar-with-dependencies diff --git a/src/main/java/org/fcrepo/bench/ActionWorker.java b/src/main/java/org/fcrepo/bench/ActionWorker.java new file mode 100644 index 0000000..c3565d7 --- /dev/null +++ b/src/main/java/org/fcrepo/bench/ActionWorker.java @@ -0,0 +1,75 @@ + +package org.fcrepo.bench; + +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.Callable; + +import org.fcrepo.bench.BenchTool.Action; +import org.fcrepo.bench.BenchTool.FedoraVersion; + +public class ActionWorker implements Callable { + + private final FedoraRestClient fedora; + + private final long binarySize; + + private final Action action; + + private final String pid; + + public ActionWorker(Action action, URI fedoraUri, String pid, long binarySize, + FedoraVersion version) { + super(); + this.binarySize = binarySize; + this.fedora = FedoraRestClient.createClient(fedoraUri, version); + this.action = action; + this.pid = pid; + } + + /* + * (non-Javadoc) + * @see java.util.concurrent.Callable#call() + */ + @Override + public BenchToolResult call() throws Exception { + /* check the action and run the appropriate test */ + switch (this.action) { + case INGEST: + return doIngest(); + case UPDATE: + return doUpdate(); + case READ: + return doRead(); + case DELETE: + return doDelete(); + default: + throw new IllegalArgumentException("The Action " + + action.name() + + " is not available in the worker thread"); + } + } + + private BenchToolResult doDelete() throws IOException { + long duration = fedora.deleteDatastream(pid); + return new BenchToolResult(-1f, duration, binarySize); + } + + private BenchToolResult doRead() throws IOException { + long duration = fedora.retrieveDatastream(pid); + float tp = binarySize * 1000f / duration; + return new BenchToolResult(tp, duration, binarySize); + } + + private BenchToolResult doUpdate() throws IOException { + long duration = fedora.updateDatastream(pid,binarySize); + float tp = binarySize * 1000f / duration; + return new BenchToolResult(tp, duration, binarySize); + } + + private BenchToolResult doIngest() throws IOException { + long duration = fedora.createDatastream(pid, binarySize); + float tp = binarySize * 1000f / duration; + return new BenchToolResult(tp, duration, binarySize); + } +} diff --git a/src/main/java/org/fcrepo/bench/BenchTool.java b/src/main/java/org/fcrepo/bench/BenchTool.java new file mode 100644 index 0000000..43c74e0 --- /dev/null +++ b/src/main/java/org/fcrepo/bench/BenchTool.java @@ -0,0 +1,209 @@ +/** + * + */ + +package org.fcrepo.bench; + +import java.io.IOException; +import java.net.URI; + +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultRedirectStrategy; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.uncommons.maths.random.XORShiftRNG; + +/** + * @author frank asseg + * + */ +public class BenchTool { + + private static final Logger LOG = LoggerFactory.getLogger(BenchTool.class); + + public static final XORShiftRNG RNG = new XORShiftRNG(); + + public static final byte[] RANDOM_SLICE = new byte[65535]; + + static { + RNG.nextBytes(RANDOM_SLICE); + } + + /* should be used by all the threads */ + static CloseableHttpClient httpClient; + + enum Action { + INGEST, READ, UPDATE, DELETE, LIST; + } + + enum FedoraVersion { + FCREPO3, FCREPO4; + } + + public static void main(String[] args) { + /* setup the command line options */ + Options ops = createOptions(); + + /* set the defaults */ + int numBinaries = 1; + long size = 1024; + int numThreads = 1; + Action action = Action.INGEST; + URI fedoraUri = URI.create("http://localhost:8080"); + String logPath = "durations.log"; + + /* and get the individual settings from the command line */ + CommandLineParser parser = new BasicParser(); + try { + CommandLine cli = parser.parse(ops, args); + if (cli.hasOption("h")) { + printUsage(ops); + return; + } + if (cli.hasOption("f")) { + String uri = cli.getOptionValue("f"); + uri = uri.replaceAll("/*$", ""); + fedoraUri = URI.create(uri); + } + if (cli.hasOption("n")) { + numBinaries = Integer.parseInt(cli.getOptionValue("n")); + } + if (cli.hasOption("s")) { + size = Long.parseLong(cli.getOptionValue("s")); + } + if (cli.hasOption("a")) { + action = Action.valueOf(cli.getOptionValue("a").toUpperCase()); + } + if (cli.hasOption("t")) { + numThreads = Integer.parseInt(cli.getOptionValue("t")); + } + if (cli.hasOption("l")) { + logPath = cli.getOptionValue("l"); + } + final HttpClientBuilder clientBuilder = + HttpClients.custom().setRedirectStrategy( + new DefaultRedirectStrategy()).setRetryHandler( + new StandardHttpRequestRetryHandler(0, false)); + if (cli.hasOption("u")) { + BasicCredentialsProvider cred = new BasicCredentialsProvider(); + cred.setCredentials(new AuthScope(fedoraUri.getHost(), + fedoraUri.getPort()), new UsernamePasswordCredentials( + cli.getOptionValue('u'), cli.getOptionValue('p'))); + clientBuilder.setDefaultCredentialsProvider(cred); + + } + httpClient = clientBuilder.build(); + + } catch (ParseException e) { + LOG.error("Unable to parse command line", e); + } + + try { + /* start the benchmark runner with the given parameters */ + FCRepoBenchRunner runner = + new FCRepoBenchRunner(getFedoraVersion(fedoraUri), + fedoraUri, action, numBinaries, size, numThreads, + logPath); + runner.runBenchmark(); + } catch (IOException e) { + LOG.error("Unable to connect to a Fedora instance at {}", fedoraUri); + } + } + + @SuppressWarnings("static-access") + private static Options createOptions() { + Options ops = new Options(); + ops.addOption(OptionBuilder + .withArgName("fedora-url") + .withDescription( + "The URL of the Fedora instance. The url must include the context path of the webapp. " + + "[default=http://localhost:8080]") + .withLongOpt("fedora-url").hasArg().create('f')); + ops.addOption(OptionBuilder.withArgName("num-actions").withDescription( + "The number of actions performed. [default=1]").withLongOpt( + "num-actions").hasArg().create('n')); + ops.addOption(OptionBuilder.withArgName("size").withDescription( + "The size of the individual binaries used. [default=1024]") + .withLongOpt("size").hasArg().create('s')); + ops.addOption(OptionBuilder + .withArgName("num-threads") + .withDescription( + "The number of threads used for performing all actions. [default=1]") + .withLongOpt("num-threads").hasArg().create('t')); + ops.addOption(OptionBuilder.withArgName("user").withDescription( + "The fedora user name").withLongOpt("user").hasArg() + .create('u')); + ops.addOption(OptionBuilder.withArgName("password").withDescription( + "The user's password").withLongOpt("password").hasArg().create( + 'p')); + ops.addOption(OptionBuilder + .withArgName("action") + .withDescription( + "The action to perform. Can be one of ingest, read, update or delete. [default=ingest]") + .withLongOpt("action").hasArg().create('a')); + ops.addOption(OptionBuilder + .withArgName("log") + .withDescription( + "The log file to which the durations will get written. [default=durations.log]") + .withLongOpt("log").hasArg().create('l')); + ops.addOption("h", "help", false, "print the help screen"); + return ops; + } + + private static FedoraVersion getFedoraVersion(URI fedoraUri) + throws IOException { + /* try to determine the Fedora Version using a GET */ + final HttpGet get = new HttpGet(fedoraUri); + final HttpResponse resp = httpClient.execute(get); + + if (resp.getStatusLine().getStatusCode() != 200) { + throw new IOException("Unable to find Fedora Server at URI " + + fedoraUri); + } + + /* + * just check the html response for a characteristic String to determine + * the fedora version + */ + String html = EntityUtils.toString(resp.getEntity()); + get.releaseConnection(); + if (html.contains("") && + html.contains("Redirecting...") && + html.contains("Redirecting...")) { + /* this seems to be a Fedora 3 instance */ + LOG.info("Found Fedora 3 at " + fedoraUri); + return FedoraVersion.FCREPO3; + } else if (html + .contains("Fedora Commons Repository 4.0") && + html.contains("You probably want to visit something a little more interesting, such as:") && + html.contains("the Fedora REST API endpoint")) { + /* this seems to be a Fedora 4 instance */ + LOG.info("Found Fedora 4 at " + fedoraUri); + return FedoraVersion.FCREPO4; + } else { + throw new IOException("Unabele to determine Fedora version at " + + fedoraUri); + } + } + + public static void printUsage(Options ops) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("BenchTool", ops); + } +} diff --git a/src/main/java/org/fcrepo/bench/BenchToolEntity.java b/src/main/java/org/fcrepo/bench/BenchToolEntity.java new file mode 100644 index 0000000..a257ae7 --- /dev/null +++ b/src/main/java/org/fcrepo/bench/BenchToolEntity.java @@ -0,0 +1,28 @@ +/** + * + */ +package org.fcrepo.bench; + +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; + + +/** + * @author frank asseg + * + */ +public class BenchToolEntity extends InputStreamEntity{ + + + public BenchToolEntity(long size, byte[] slice) { + super(new BenchToolInputStream(size, slice), size, ContentType.APPLICATION_OCTET_STREAM); + } + + /* (non-Javadoc) + * @see org.apache.http.entity.InputStreamEntity#isRepeatable() + */ + @Override + public boolean isRepeatable() { + return true; + } +} diff --git a/src/main/java/org/fcrepo/bench/BenchToolFC3.java b/src/main/java/org/fcrepo/bench/BenchToolFC3.java deleted file mode 100644 index 1834bfa..0000000 --- a/src/main/java/org/fcrepo/bench/BenchToolFC3.java +++ /dev/null @@ -1,274 +0,0 @@ -/** - * - */ - -package org.fcrepo.bench; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import org.uncommons.maths.random.XORShiftRNG; - -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpResponse; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.client.DefaultHttpClient; - -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.apache.http.HttpHost; -import org.apache.http.client.AuthCache; -import org.apache.http.client.protocol.ClientContext; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.protocol.BasicHttpContext; - -/** - * Fedora 3 Benchmarking Tool - * @author frank asseg - * - */ -public class BenchToolFC3 { - - private static final DecimalFormat FORMATTER = new DecimalFormat("000.00"); - - private final DefaultHttpClient client = new DefaultHttpClient(); - - private final URI fedoraUri; - - private final OutputStream ingestOut; - private final BasicHttpContext authContext; - - public BenchToolFC3(String fedoraUri, String user, String pass) - throws IOException { - super(); - ingestOut = new FileOutputStream("ingest.log"); - if (fedoraUri.charAt(fedoraUri.length() - 1) == '/') { - fedoraUri = fedoraUri.substring(0, fedoraUri.length() - 1); - } - this.fedoraUri = URI.create(fedoraUri); - this.client.getCredentialsProvider().setCredentials( - new AuthScope(this.fedoraUri.getHost(), this.fedoraUri - .getPort()), - new UsernamePasswordCredentials(user, pass)); - - // setup authcache to enable pre-emptive auth - AuthCache authCache = new BasicAuthCache(); - BasicScheme basicAuth = new BasicScheme(); - authCache.put(new HttpHost(this.fedoraUri.getHost(),this.fedoraUri.getPort()), basicAuth); - this.authContext = new BasicHttpContext(); - authContext.setAttribute(ClientContext.AUTH_CACHE, authCache); - } - - private String ingestObject(String label) throws Exception { - HttpPost post = - new HttpPost( - fedoraUri.toASCIIString() + - "/objects/new?format=info:fedora/fedora-system:FOXML-1.1&label=" + - label); - HttpResponse resp = client.execute(post,authContext); - String answer = IOUtils.toString(resp.getEntity().getContent()); - post.releaseConnection(); - - if (resp.getStatusLine().getStatusCode() != 201) { - throw new Exception("Unable to ingest object, fedora returned " + - resp.getStatusLine().getStatusCode()); - } - return answer; - } - - private void ingestDatastream(String objectId, String label, long size) - throws Exception { - HttpPost post = new HttpPost(fedoraUri.toASCIIString() + "/objects/" - + objectId + "/datastreams/" + label - + "?versionable=true&controlGroup=M"); - post.setHeader("Content-Type", "application/octet-stream"); - post.setEntity(new InputStreamEntity(new BenchToolInputStream(size),size)); - long start = System.currentTimeMillis(); - HttpResponse resp = client.execute(post,authContext); - IOUtils.write((System.currentTimeMillis() - start) + "\n", ingestOut); - post.releaseConnection(); - if (resp.getStatusLine().getStatusCode() != 201) { - throw new Exception("Unable to ingest datastream " + label - + " fedora returned " + resp.getStatusLine()); - } - } - - private void updateDatastream(String objectId, String label, long size) - throws Exception { - HttpPut put = new HttpPut(fedoraUri.toASCIIString() + "/objects/" - + objectId + "/datastreams/" + label - + "?versionable=true&controlGroup=M"); - put.setHeader("Content-Type", "application/octet-stream"); - put.setEntity(new InputStreamEntity(new BenchToolInputStream(size),size)); - long start = System.currentTimeMillis(); - HttpResponse resp = client.execute(put,authContext); - IOUtils.write((System.currentTimeMillis() - start) + "\n", ingestOut); - put.releaseConnection(); - if (resp.getStatusLine().getStatusCode() != 200) { - throw new Exception("Unable to update datastream " + label - + " fedora returned " + resp.getStatusLine()); - } - } - - private void readDatastream(String objectId, String label) - throws Exception { - // read properties - String objCreated = readProperty(objectId, null, "objCreateDate"); - String dsCreated = readProperty(objectId, "ds-1", "dsCreateDate"); - - // read datastream content - HttpGet get = new HttpGet(fedoraUri.toASCIIString() + "/objects/" - + objectId + "/datastreams/" + label + "/content"); - long start = System.currentTimeMillis(); - HttpResponse resp = client.execute(get,authContext); - InputStream in = resp.getEntity().getContent(); - byte[] buf = new byte[8192]; - for ( int read = -1; (read = in.read(buf)) != -1; ) { } - IOUtils.write((System.currentTimeMillis() - start) + "\n", ingestOut); - get.releaseConnection(); - - if (resp.getStatusLine().getStatusCode() != 200) { - throw new Exception("Unable to update datastream " + label - + " fedora returned " + resp.getStatusLine()); - } - } - - private void deleteObject(String objectId) throws Exception { - HttpDelete del = new HttpDelete(fedoraUri.toASCIIString() + "/objects/" - + objectId ); - long start = System.currentTimeMillis(); - HttpResponse resp = client.execute(del,authContext); - IOUtils.write((System.currentTimeMillis() - start) + "\n", ingestOut); - del.releaseConnection(); - if (resp.getStatusLine().getStatusCode() != 200) { - throw new Exception("Unable to delete object fedora returned " - + resp.getStatusLine()); - } - } - - private List listObjects(int numObjects) throws Exception { - String uri = fedoraUri.toASCIIString() + "/objects?terms=test*" - + "&resultFormat=xml&pid=true&maxResults="+numObjects; - HttpGet get = new HttpGet(uri); - HttpResponse resp = client.execute(get,authContext); - - List pids = new ArrayList(); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse( resp.getEntity().getContent() ); - NodeList nodeList = document.getElementsByTagName("pid"); - for ( int i = 0; i < nodeList.getLength() && i < numObjects; i++ ) { - Node n = nodeList.item(i); - pids.add( n.getTextContent() ); - } - get.releaseConnection(); - return pids; - } - private String readProperty(String pid, String dsName, String property) throws Exception { - String uri = fedoraUri.toASCIIString() + "/objects/" + pid; - if ( dsName != null ) { uri += "/datastreams/" + dsName; } - uri += "?format=xml"; - HttpGet get = new HttpGet(uri); - HttpResponse resp = client.execute(get,authContext); - - String value = null; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse( resp.getEntity().getContent() ); - NodeList nodeList = document.getElementsByTagName(property); - for ( int i = 0; i < nodeList.getLength() && value == null; i++ ) { - Node n = nodeList.item(i); - value = n.getTextContent(); - } - get.releaseConnection(); - return value; - } - - private void shutdown() { - IOUtils.closeQuietly(ingestOut); - } - - public static void main(String[] args) { - String uri = args[0]; - String user = args[1]; - String pass = args[2]; - int numObjects = Integer.parseInt(args[3]); - long size = Integer.parseInt(args[4]); - String action = "ingest"; - if ( args.length > 5 ) { action = args[5]; } - BenchToolFC3 bench = null; - if ( action != null && action.equals("delete") ) { - System.out.println("deleting " + numObjects + " objects"); - } else if ( action != null && action.equals("update") ) { - System.out.println("updating " + numObjects - + " objects with datastream size " + size); - } else if ( action != null && action.equals("read") ) { - System.out.println("reading " + numObjects - + " objects with datastream size " + size); - } else { - System.out.println("ingesting " + numObjects - + " objects with datastream size " + size); - } - Random rnd; - if ( "java.util.Random".equals(System.getProperty("random.impl")) ) { - rnd = new Random(); - } else { - rnd = new XORShiftRNG(); - } - - List pids = null; - long start = System.currentTimeMillis(); - try { - bench = new BenchToolFC3(uri, user, pass); - for (int i = 0; i < numObjects; i++) { - String objectId = null; - if ( action != null && (action.equals("delete") || action.equals("update") || action.equals("read")) ) { - if ( pids == null ) { - pids = bench.listObjects(numObjects); - } - objectId = pids.get( rnd.nextInt(pids.size()) ); - if ( action.equals("delete") ) { - pids.remove(objectId); - bench.deleteObject(objectId); - } else if ( action.equals("read") ) { - bench.readDatastream(objectId, "ds-1"); - } else { - bench.updateDatastream(objectId, "ds-1", size); - } - } else { - objectId = bench.ingestObject("test-" +i); - bench.ingestDatastream(objectId, "ds-1", size); - } - float percent = (float) (i + 1) / (float) numObjects * 100f; - System.out.print("\r" + FORMATTER.format(percent) + "%"); - } - long duration = System.currentTimeMillis() - start; - System.out.println(" - " + action + " finished"); - System.out.println("Complete " + action + " of " + numObjects + " files took " + duration + " ms\n"); - System.out.println("throughput was " + FORMATTER.format((double) numObjects * (double) size /1024d / duration) + " mb/s\n"); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - bench.shutdown(); - } - - } -} diff --git a/src/main/java/org/fcrepo/bench/BenchToolFC4.java b/src/main/java/org/fcrepo/bench/BenchToolFC4.java deleted file mode 100644 index b2ee1c4..0000000 --- a/src/main/java/org/fcrepo/bench/BenchToolFC4.java +++ /dev/null @@ -1,311 +0,0 @@ -/** - * - */ - -package org.fcrepo.bench; - -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.List; -import java.nio.charset.Charset; -import java.util.Random; -import org.uncommons.maths.random.XORShiftRNG; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.LineIterator; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.client.DefaultHttpClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.hp.hpl.jena.rdf.model.Model; -import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.RDFNode; -import com.hp.hpl.jena.rdf.model.StmtIterator; -import com.hp.hpl.jena.rdf.model.NodeIterator; -import com.hp.hpl.jena.rdf.model.Property; -import com.hp.hpl.jena.rdf.model.Resource; - -/** - * Fedora 4 Benchmarking Tool - * @author frank asseg - * - */ -public class BenchToolFC4 { - - private static final DecimalFormat FORMATTER = new DecimalFormat("000.00"); - - private static int numThreads = 0; - - private static int maxThreads = 15; - - - private static final Logger LOG = - LoggerFactory.getLogger(BenchToolFC4.class); - - - private static int getClusterSize(String uri) throws IOException { - final Model model = ModelFactory.createDefaultModel(); - model.read(uri + "/rest"); - StmtIterator it = model.listStatements(model.createResource(uri + "/rest/") ,model.createProperty("http://fedora.info/definitions/v4/repository#clusterSize"), (RDFNode) null); - if (!it.hasNext()) { - return 0; - } - return Integer.parseInt(it.next().getObject().asLiteral().getString()); - - } - - private static List listObjects(String uri,int max) throws IOException { - DefaultHttpClient client = new DefaultHttpClient(); - HttpGet get = new HttpGet(uri); - HttpResponse resp = client.execute(get); - List pids = new ArrayList(); - Model m = ModelFactory.createDefaultModel(); - m.read( resp.getEntity().getContent(), null, "TURTLE" ); - Property hasChild = m.createProperty( - "http://fedora.info/definitions/v4/repository#", "hasChild"); - NodeIterator childNodes = m.listObjectsOfProperty(hasChild); - while ( childNodes.hasNext() ) { - RDFNode child = childNodes.next(); - if ( child.toString().startsWith(uri) - && child.toString().length() > uri.length() ) { - pids.add( child.toString().replaceAll(".*" + uri + "/","") ); - } - if ( pids.size() >= max ) { break; } - } - get.releaseConnection(); - return pids; - } - private static Model readProperties(String uri) throws IOException { - DefaultHttpClient client = new DefaultHttpClient(); - HttpGet get = new HttpGet(uri); - HttpResponse resp = client.execute(get); - Model m = ModelFactory.createDefaultModel(); - m.read( resp.getEntity().getContent(), null, "TURTLE" ); - get.releaseConnection(); - return m; - } - - public static void main(String[] args) { - String uri = args[0]; - int numObjects = Integer.parseInt(args[1]); - long size = Long.parseLong(args[2]); - maxThreads = Integer.parseInt(args[3]); - String action = "ingest"; - if ( args.length > 4 ) { action = args[4]; } - BenchToolFC4 bench = null; - if ( action != null && action.equals("delete") ) { - LOG.info("deleting {} objects", numObjects); - } else if ( action != null && action.equals("update") ) { - LOG.info("updating {} objects with datastream size {}", numObjects, size); - } else if ( action != null && action.equals("read") ) { - LOG.info("reading {} objects with datastream size {}", numObjects, size); - } else { - LOG.info("ingesting {} objects with datastream size {}", numObjects, size); - } - - FileOutputStream ingestOut = null; - List pids = null; - Random rnd = null; - if ( "java.util.Random".equals(System.getProperty("random.impl")) ) { - rnd = new Random(); - } else { - rnd = new XORShiftRNG(); - } - try { - final int initialClusterSize = getClusterSize(uri); - LOG.info("Initial cluster size is {}", initialClusterSize); - ingestOut = new FileOutputStream("ingest.log"); - long start = System.currentTimeMillis(); - for (int i = 0; i < numObjects; i++) { - while (numThreads >= maxThreads) { - Thread.sleep(10); - } - String pid = "benchfc4-" + start + "-" + (i + 1); - if ( action != null && - (action.equals("update") || action.equals("delete") || action.equals("read")) ) { - if ( pids == null ) { - pids = listObjects(uri + "/rest/objects",numObjects); - } - pid = pids.get( rnd.nextInt(pids.size()) ); - if ( action.equals("delete") ) { pids.remove(pid); } - } - Thread t = new Thread(new Ingester(uri, ingestOut, pid, size, - action)); - t.start(); - numThreads++; - float percent = (float) (i + 1) / (float) numObjects * 100f; - System.out.print("\r" + FORMATTER.format(percent) + "%"); - } - while (numThreads > 0) { - Thread.sleep(100); - } - long duration = System.currentTimeMillis() - start; - System.out.println(" - " + action + " finished"); - LOG.info("Processing {} objects took {} ms", numObjects, duration); - final int endClusterSize = getClusterSize(uri); - if (initialClusterSize != endClusterSize) { - LOG.warn("Initial cluster size was {} but the cluster had size {} at the end", initialClusterSize, endClusterSize); - } - LOG.info("Throughput was {} mb/s",FORMATTER.format((double) numObjects * (double) size / - 1024d / duration)); - } catch (Exception e) { - e.printStackTrace(); - } finally { - IOUtils.closeQuietly(ingestOut); - } - - } - - private static class Ingester implements Runnable { - - private final DefaultHttpClient client = new DefaultHttpClient(); - - private final URI fedoraUri; - - private final OutputStream ingestOut; - - private final long size; - - private final String pid; - - private final String action; - - public Ingester( String fedoraUri, OutputStream out, String pid, - long size, String action) throws IOException { - super(); - ingestOut = out; - if (fedoraUri.charAt(fedoraUri.length() - 1) == '/') { - fedoraUri = fedoraUri.substring(0, fedoraUri.length() - 1); - } - this.fedoraUri = URI.create(fedoraUri); - this.size = size; - this.pid = pid; - this.action = action; - } - - @Override - public void run() { - try { - if ( action != null && action.equals("delete") ) { - this.deleteObject(); - } else if ( action != null && action.equals("update") ) { - this.updateObject(); - } else if ( action != null && action.equals("read") ) { - this.readObject(); - } else { - this.ingestObject(); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - private void ingestObject() throws Exception { - HttpPost post = - new HttpPost(fedoraUri.toASCIIString() + "/rest/objects/" + - pid + "/DS1/fcr:content"); - post.setHeader("Content-Type", "application/octet-stream"); - post.setEntity(new InputStreamEntity(new BenchToolInputStream(size), size)); - long start = System.currentTimeMillis(); - HttpResponse resp = client.execute(post); - String answer = IOUtils.toString(resp.getEntity().getContent()); - post.releaseConnection(); - - if (resp.getStatusLine().getStatusCode() != 201) { - LOG.error(answer); - BenchToolFC4.numThreads--; - throw new Exception( - "Unable to ingest object, fedora returned " + - resp.getStatusLine().getStatusCode()); - } - IOUtils.write((System.currentTimeMillis() - start) + "\n", - ingestOut); - BenchToolFC4.numThreads--; - } - - private void readObject() throws Exception { - long start = System.currentTimeMillis(); - - // read properties - String objURI = fedoraUri.toASCIIString() + "/rest/objects/" + pid; - Model m = readProperties( objURI ); - Property created = m.createProperty( - "http://fedora.info/definitions/v4/repository#", "created"); - Resource objRes = m.getResource( objURI ); - Resource dsRes = m.getResource( objURI + "/DS1" ); - String objCreated = m.getProperty( objRes, created ).getString(); - String dsCreated = m.getProperty( dsRes, created ).getString(); - - // read datastream content - HttpGet get = - new HttpGet(fedoraUri.toASCIIString() + "/rest/objects/" + - pid + "/DS1/fcr:content"); - HttpResponse resp = client.execute(get); - InputStream in = resp.getEntity().getContent(); - byte[] buf = new byte[8192]; - for ( int read = -1; (read = in.read(buf)) != -1; ) { } - get.releaseConnection(); - - if (resp.getStatusLine().getStatusCode() != 200) { - BenchToolFC4.numThreads--; - throw new Exception( - "Unable to read object: " + pid + ", fedora returned " + - resp.getStatusLine().getStatusCode()); - } - IOUtils.write((System.currentTimeMillis() - start) + "\n", - ingestOut); - BenchToolFC4.numThreads--; - } - - private void updateObject() throws Exception { - HttpPut put = - new HttpPut(fedoraUri.toASCIIString() + "/rest/objects/" + - pid + "/DS1/fcr:content"); - put.setHeader("Content-Type", "application/octet-stream"); - put.setEntity(new InputStreamEntity(new BenchToolInputStream(size),size)); - long start = System.currentTimeMillis(); - HttpResponse resp = client.execute(put); - put.releaseConnection(); - - if (resp.getStatusLine().getStatusCode() != 204) { - BenchToolFC4.numThreads--; - throw new Exception( - "Unable to ingest object, fedora returned " + - resp.getStatusLine().getStatusCode()); - } - IOUtils.write((System.currentTimeMillis() - start) + "\n", - ingestOut); - BenchToolFC4.numThreads--; - } - - private void deleteObject() throws Exception { - HttpDelete del = new HttpDelete(fedoraUri.toASCIIString() - + "/rest/objects/" + pid ); - long start = System.currentTimeMillis(); - HttpResponse resp = client.execute(del); - del.releaseConnection(); - - if (resp.getStatusLine().getStatusCode() != 204) { - BenchToolFC4.numThreads--; - throw new Exception( - "Unable to delete object, fedora returned " + - resp.getStatusLine().getStatusCode()); - } - IOUtils.write((System.currentTimeMillis() - start) + "\n", - ingestOut); - BenchToolFC4.numThreads--; - } - - } -} diff --git a/src/main/java/org/fcrepo/bench/BenchToolInputStream.java b/src/main/java/org/fcrepo/bench/BenchToolInputStream.java index 0b6c7b0..c9f1698 100644 --- a/src/main/java/org/fcrepo/bench/BenchToolInputStream.java +++ b/src/main/java/org/fcrepo/bench/BenchToolInputStream.java @@ -1,47 +1,47 @@ -/** - * - */ + package org.fcrepo.bench; import java.io.IOException; import java.io.InputStream; -import java.util.Random; -import org.uncommons.maths.random.XORShiftRNG; - - -/** - * @author frank asseg - * - */ public class BenchToolInputStream extends InputStream { private final long size; - private long idx = 0; - private final Random rng; + private long bytesRead; + private final byte[] slice; + private final int sliceLen; + private int slicePos; - public BenchToolInputStream(long size) { + public BenchToolInputStream(long size, byte[] slice) { super(); this.size = size; + this.slice = slice; + this.sliceLen = slice.length; + } - if ( "java.util.Random".equals(System.getProperty("random.impl")) ) { - /* standard jdk random */ - rng = new Random(); - } else { - /* quite the fast RNG from uncommons-math */ - rng = new XORShiftRNG(); + @Override + public int read() throws IOException { + if (slicePos == 0 || slicePos == sliceLen - 1) { + slicePos = BenchTool.RNG.nextInt((int) Math.floor(sliceLen /2f)); } + return slice[slicePos++]; } - /* (non-Javadoc) - * @see java.io.InputStream#read() - */ @Override - public int read() throws IOException { - if (idx++ <= size) { - return rng.nextInt(); - }else { - throw new IOException("Inputstream size limit reached"); + public int read(byte[] b) throws IOException { + int i=0; + for (;i results = Collections + .synchronizedList(new ArrayList()); + + private final FedoraVersion version; + + private final URI fedoraUri; + + private final Action action; + + private final int numBinaries; + + private final long size; + + private final int numThreads; + + private final ExecutorService executor; + + private final FedoraRestClient fedora; + + private FileOutputStream logOut; + + public FCRepoBenchRunner(FedoraVersion version, URI fedoraUri, + Action action, int numBinaries, long size, int numThreads, + String logpath) { + super(); + this.version = version; + this.fedoraUri = fedoraUri; + this.action = action; + this.numBinaries = numBinaries; + this.size = size; + this.numThreads = numThreads; + this.executor = Executors.newFixedThreadPool(numThreads); + this.fedora = FedoraRestClient.createClient(fedoraUri, version); + try { + this.logOut = new FileOutputStream(logpath); + } catch (FileNotFoundException e) { + this.logOut = null; + LOG.warn( + "Unable to open log file at {}. No log output will be generated", + logpath); + } + } + + private int getClusterSize() { + final Model model = ModelFactory.createDefaultModel(); + model.read(this.fedoraUri + "/rest"); + StmtIterator it = + model.listStatements( + model.createResource(fedoraUri + "/rest/"), + model.createProperty("http://fedora.info/definitions/v4/repository#clusterSize"), + (RDFNode) null); + if (!it.hasNext()) { + return 0; + } + return Integer.parseInt(it.next().getObject().asLiteral().getString()); + } + + public void runBenchmark() { + this.logParameters(); + /* + * first create the required top level objects so their creation won't + * affect the pure action performance + */ + final List pids = prepareObjects(); + + LOG.info("scheduling {} actions", numBinaries); + + /* schedule all the action workers for execution */ + final List> futures = new ArrayList<>(); + for (String pid : pids) { + futures.add(executor.submit(new ActionWorker(action, fedoraUri, pid, size, version))); + } + + /* retrieve the workers' results */ + try { + this.fetchResults(futures); + } catch (InterruptedException | ExecutionException | IOException e) { + LOG.error("Error while getting results from worker threads",e); + }finally{ + this.executor.shutdown(); + } + + /* delete all the created objects and datastreams from the repository */ + this.purgeObjects(pids); + + this.logResults(); + } + + private void logParameters() { + LOG.info( + "Running {} {} action(s) against {} with a binary size of {} using {} thread(s)", + new Object[] {numBinaries, action.name(), version.name(), size, + numThreads}); + if (version == FedoraVersion.FCREPO4) { + LOG.info("The Fedora cluster has {} node(s) before the benchmark", + getClusterSize()); + } + } + + private void logResults() { + long duration = 0; + long numBytes = 0; + for (BenchToolResult res : results) { + duration = duration + res.getDuration(); + numBytes = numBytes + res.getSize(); + } + float throughputPerThread = 0f; + throughputPerThread = size * numBinaries * 1000f / (1024f * 1024f * duration); + + /* now the bench is finished and the result will be printed out */ + LOG.info("Completed {} {} action(s) executed in {} ms", new Object[] { + this.numBinaries, action, duration}); + if (version == FedoraVersion.FCREPO4) { + LOG.info("The Fedora cluster has {} node(s) after the benchmark", + getClusterSize()); + } + if (numThreads == 1) { + LOG.info("Throughput was {} MB/sec", FORMAT + .format(throughputPerThread)); + } else { + LOG.info("Throughput was {} MB/sec", FORMAT + .format(throughputPerThread * numThreads)); + LOG.info("Throughput per thread was {} MB/sec", FORMAT + .format(throughputPerThread)); + } + } + + private List fetchResults(List> futures) throws InterruptedException, ExecutionException, IOException { + int count = 0; + for (Future f : futures) { + BenchToolResult res = f.get(); + LOG.debug("{} of {} actions finished", ++count, numBinaries); + if (logOut != null) { + logOut.write((res.getDuration() + "\n").getBytes()); + } + results.add(res); + } + return results; + } + + private void purgeObjects(List pids) { + LOG.info("purging {} objects and datastreams", numBinaries); + fedora.purgeObjects(pids,action != Action.DELETE); + } + + private List prepareObjects() { + final List pids = new ArrayList<>(); + LOG.info("preparing {} objects", numBinaries); + for (int i = 0; i < numBinaries; i++) { + pids.add(UUID.randomUUID().toString()); + } + fedora.createObjects(pids); + if (this.action == Action.UPDATE || this.action == Action.READ || this.action == Action.DELETE) { + LOG.info("preparing {} datastreams of size {} for {}", new Object[] {numBinaries, size, action}); + // add datastreams in preparation which can be manipulated + fedora.createDatastreams(pids, size); + } + return pids; + } +} diff --git a/src/main/java/org/fcrepo/bench/Fedora3RestClient.java b/src/main/java/org/fcrepo/bench/Fedora3RestClient.java new file mode 100644 index 0000000..f0b12df --- /dev/null +++ b/src/main/java/org/fcrepo/bench/Fedora3RestClient.java @@ -0,0 +1,131 @@ +/** + * + */ + +package org.fcrepo.bench; + +import java.io.IOException; +import java.net.URI; + +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.util.EntityUtils; +import org.fcrepo.bench.BenchTool.FedoraVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author frank asseg + * + */ +public class Fedora3RestClient extends FedoraRestClient { + + private static final Logger LOG = LoggerFactory + .getLogger(Fedora3RestClient.class); + + public Fedora3RestClient(URI fedoraUri) { + super(fedoraUri, FedoraVersion.FCREPO3); + } + + @Override + protected long createObject(String pid) throws IOException { + HttpPost post = + new HttpPost(this.fedoraUri + "/objects/bt:" + pid + + "?format=info:fedora/fedora-system:FOXML-1.1&label=" + + pid); + long time = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(post); + long duration = System.currentTimeMillis() - time; + post.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 201) { + throw new IOException("Unable to create object at /objects/" + pid + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + return duration; + } + + @Override + protected long createDatastream(String pid, long size) throws IOException { + String dsUri = + this.fedoraUri + "/objects/bt:" + pid + + "/datastreams/ds1?versionable=true&controlGroup=M"; + HttpPost post = new HttpPost(dsUri); + post.setEntity(new BenchToolEntity(size, BenchTool.RANDOM_SLICE)); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(post); + long duration = System.currentTimeMillis() - start; + if (resp.getStatusLine().getStatusCode() != 201) { + throw new IOException("Unable to create datastream at " + dsUri + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + post.releaseConnection(); + return duration; + } + + @Override + protected long updateDatastream(String pid, long size) throws IOException { + String dsUri = + this.fedoraUri + "/objects/bt:" + pid + + "/datastreams/ds1?versionable=true&controlGroup=M"; + HttpPut put = new HttpPut(dsUri); + put.setEntity(new BenchToolEntity(size, BenchTool.RANDOM_SLICE)); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(put); + long duration = System.currentTimeMillis() - start; + put.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 200) { + throw new IOException("Unable to update datastream at " + dsUri + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + return duration; + } + + @Override + protected long retrieveDatastream(String pid) throws IOException { + String dsUri = + this.fedoraUri + "/objects/bt:" + pid + + "/datastreams/ds1/content"; + HttpGet get = new HttpGet(dsUri); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(get); + long duration = System.currentTimeMillis() - start; + EntityUtils.consume(resp.getEntity()); + get.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 200) { + throw new IOException("Unable to retrieve datastream from " + + dsUri + "\nFedora returned " + + resp.getStatusLine().getStatusCode()); + } + return duration; + } + + @Override + protected long deleteObject(String pid) throws IOException { + HttpDelete delete = + new HttpDelete(this.fedoraUri + "/objects/bt:" + pid); + long time = System.currentTimeMillis(); + BenchTool.httpClient.execute(delete); + long duration = System.currentTimeMillis() - time; + delete.releaseConnection(); + return duration; + } + + @Override + protected long deleteDatastream(String pid) throws IOException { + String dsUri = + this.fedoraUri + "/objects/bt:" + pid + "/datastreams/ds1"; + HttpDelete delete = new HttpDelete(dsUri); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(delete); + long duration = System.currentTimeMillis() - start; + delete.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 200) { + throw new IOException("Unable to delete datastream from " + dsUri + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + return duration; + } +} diff --git a/src/main/java/org/fcrepo/bench/Fedora4RestClient.java b/src/main/java/org/fcrepo/bench/Fedora4RestClient.java new file mode 100644 index 0000000..073a05e --- /dev/null +++ b/src/main/java/org/fcrepo/bench/Fedora4RestClient.java @@ -0,0 +1,122 @@ +/** + * + */ + +package org.fcrepo.bench; + +import java.io.IOException; +import java.net.URI; + +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.fcrepo.bench.BenchTool.FedoraVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author frank asseg + * + */ +public class Fedora4RestClient extends FedoraRestClient { + + private static final Logger LOG = LoggerFactory + .getLogger(Fedora4RestClient.class); + + public Fedora4RestClient(URI fedoraUri) { + super(fedoraUri, FedoraVersion.FCREPO4); + } + + @Override + protected long createObject(String pid) throws IOException { + HttpPost post = new HttpPost(this.fedoraUri + "/rest/objects/" + pid); + long time = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(post); + long duration = System.currentTimeMillis() - time; + post.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 201) { + throw new IOException("Unable to create object at /objects/" + pid + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + return duration; + } + + @Override + protected long createDatastream(String pid, long size) throws IOException { + String dsUri = + this.fedoraUri + "/rest/objects/" + pid + "/ds1/fcr:content"; + HttpPost post = new HttpPost(dsUri); + post.setEntity(new BenchToolEntity(size, BenchTool.RANDOM_SLICE)); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(post); + long duration = System.currentTimeMillis() - start; + if (resp.getStatusLine().getStatusCode() != 201) { + throw new IOException("Unable to create datastream at " + dsUri + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + post.releaseConnection(); + return duration; + } + + @Override + protected long updateDatastream(String pid, long size) throws IOException { + String dsUri = + this.fedoraUri + "/rest/objects/" + pid + "/ds1/fcr:content"; + HttpPut put = new HttpPut(dsUri); + put.setEntity(new BenchToolEntity(size, BenchTool.RANDOM_SLICE)); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(put); + long duration = System.currentTimeMillis() - start; + put.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 204) { + throw new IOException("Unable to update datastream at " + dsUri + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + return duration; + } + + @Override + protected long retrieveDatastream(String pid) throws IOException { + String dsUri = + this.fedoraUri + "/rest/objects/" + pid + "/ds1/fcr:content"; + HttpGet get = new HttpGet(dsUri); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(get); + long duration = System.currentTimeMillis() - start; + get.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 200) { + throw new IOException("Unable to retrieve datastream from " + + dsUri + "\nFedora returned " + + resp.getStatusLine().getStatusCode()); + } + return duration; + } + + @Override + protected long deleteObject(String pid) throws IOException { + HttpDelete delete = + new HttpDelete(this.fedoraUri + "/rest/objects/" + pid); + long time = System.currentTimeMillis(); + BenchTool.httpClient.execute(delete); + long duration = System.currentTimeMillis() - time; + delete.releaseConnection(); + return duration; + } + + @Override + protected long deleteDatastream(String pid) throws IOException { + String dsUri = this.fedoraUri + "/rest/objects/" + pid + "/ds1"; + HttpDelete delete = new HttpDelete(dsUri); + long start = System.currentTimeMillis(); + HttpResponse resp = BenchTool.httpClient.execute(delete); + long duration = System.currentTimeMillis() - start; + delete.releaseConnection(); + if (resp.getStatusLine().getStatusCode() != 204) { + throw new IOException("Unable to delete datastream from " + dsUri + + "\nFedora returned " + resp.getStatusLine().getStatusCode()); + } + return duration; + } +} diff --git a/src/main/java/org/fcrepo/bench/FedoraRestClient.java b/src/main/java/org/fcrepo/bench/FedoraRestClient.java new file mode 100644 index 0000000..61be709 --- /dev/null +++ b/src/main/java/org/fcrepo/bench/FedoraRestClient.java @@ -0,0 +1,80 @@ + +package org.fcrepo.bench; + +import java.io.IOException; +import java.net.URI; +import java.util.List; + +import org.fcrepo.bench.BenchTool.FedoraVersion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class FedoraRestClient { + + private static final Logger LOG = + LoggerFactory.getLogger(FedoraRestClient.class); + + private final FedoraVersion version; + + protected final URI fedoraUri; + + public FedoraRestClient(URI fedoraUri, FedoraVersion version) { + super(); + this.version = version; + this.fedoraUri = fedoraUri; + } + + protected abstract long deleteDatastream(String pid) throws IOException; + protected abstract long deleteObject(String pid) throws IOException; + protected abstract long createObject(String pid) throws IOException; + protected abstract long createDatastream(String pid, long size) throws IOException; + protected abstract long retrieveDatastream(String pid) throws IOException; + protected abstract long updateDatastream(String pid, long size) throws IOException; + + final void purgeObjects(List pids, boolean removeDatastreams) { + for (String pid : pids) { + try { + if (removeDatastreams) { + this.deleteDatastream(pid);; + } + this.deleteObject(pid); + } catch (IOException e) { + LOG.error("Unable to prepare objects in Fedora", e); + } + } + } + + final void createObjects(List pids) { + for (String pid : pids) { + try { + this.createObject(pid); + } catch (IOException e) { + LOG.error("Unable to prepare objects in Fedora", e); + } + } + } + + public void createDatastreams(List pids, long size) { + for (String pid : pids) { + try { + this.createDatastream(pid, size); + } catch (IOException e) { + LOG.error("Unable to prepare datastream in Fedora", e); + } + } + } + + public static FedoraRestClient createClient(URI fedoraUri, + FedoraVersion version) { + switch (version) { + case FCREPO3: + return new Fedora3RestClient(fedoraUri); + case FCREPO4: + return new Fedora4RestClient(fedoraUri); + default: + throw new IllegalArgumentException( + "No client available for Fedora Version" + + version.name()); + } + } +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..d5c61f4 --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,17 @@ + + + + + + + + %-9(%d{HH:mm:ss}) %-5level %msg%n + + + + + + + + + \ No newline at end of file