Clover icon

jalviewX

  1. Project Clover database Wed Oct 31 2018 15:13:58 GMT
  2. Package jalview.ext.ensembl

File EnsemblRestClient.java

 

Coverage histogram

../../../img/srcFileCovDistChart1.png
53% of files have more coverage

Code metrics

42
143
18
1
587
345
48
0.34
7.94
18
2.67

Classes

Class Line # Actions
EnsemblRestClient 52 143 48 191
0.05911335.9%
 

Contributing tests

This file is covered by 26 tests. .

Source view

1    /*
2    * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$)
3    * Copyright (C) $$Year-Rel$$ The Jalview Authors
4    *
5    * This file is part of Jalview.
6    *
7    * Jalview is free software: you can redistribute it and/or
8    * modify it under the terms of the GNU General Public License
9    * as published by the Free Software Foundation, either version 3
10    * of the License, or (at your option) any later version.
11    *
12    * Jalview is distributed in the hope that it will be useful, but
13    * WITHOUT ANY WARRANTY; without even the implied warranty
14    * of MERCHANTABILITY or FITNESS FOR A PARTICULAR
15    * PURPOSE. See the GNU General Public License for more details.
16    *
17    * You should have received a copy of the GNU General Public License
18    * along with Jalview. If not, see <http://www.gnu.org/licenses/>.
19    * The Jalview Authors are detailed in the 'AUTHORS' file.
20    */
21    package jalview.ext.ensembl;
22   
23    import jalview.bin.Jalview;
24    import jalview.io.DataSourceType;
25    import jalview.io.FileParse;
26    import jalview.util.StringUtils;
27   
28    import java.io.BufferedReader;
29    import java.io.DataOutputStream;
30    import java.io.IOException;
31    import java.io.InputStream;
32    import java.io.InputStreamReader;
33    import java.net.HttpURLConnection;
34    import java.net.MalformedURLException;
35    import java.net.ProtocolException;
36    import java.net.URL;
37    import java.util.HashMap;
38    import java.util.List;
39    import java.util.Map;
40   
41    import javax.ws.rs.HttpMethod;
42   
43    import org.json.simple.JSONArray;
44    import org.json.simple.JSONObject;
45    import org.json.simple.parser.JSONParser;
46   
47    /**
48    * Base class for Ensembl REST service clients
49    *
50    * @author gmcarstairs
51    */
 
52    abstract class EnsemblRestClient extends EnsemblSequenceFetcher
53    {
54    private static final int DEFAULT_READ_TIMEOUT = 5 * 60 * 1000; // 5 minutes
55   
56    private static final int CONNECT_TIMEOUT_MS = 10 * 1000; // 10 seconds
57   
58    private static final int MAX_RETRIES = 3;
59   
60    private static final int HTTP_OK = 200;
61   
62    private static final int HTTP_OVERLOAD = 429;
63   
64    /*
65    * update these constants when Jalview has been checked / updated for
66    * changes to Ensembl REST API (ref JAL-2105)
67    * @see https://github.com/Ensembl/ensembl-rest/wiki/Change-log
68    * @see http://rest.ensembl.org/info/rest?content-type=application/json
69    */
70    private static final String LATEST_ENSEMBLGENOMES_REST_VERSION = "6.3";
71   
72    private static final String LATEST_ENSEMBL_REST_VERSION = "6.3";
73   
74    private static final String REST_CHANGE_LOG = "https://github.com/Ensembl/ensembl-rest/wiki/Change-log";
75   
76    private static Map<String, EnsemblData> domainData;
77   
78    private final static long AVAILABILITY_RETEST_INTERVAL = 10000L; // 10 seconds
79   
80    private final static long VERSION_RETEST_INTERVAL = 1000L * 3600; // 1 hr
81   
82    protected static final String CONTENT_TYPE_JSON = "?content-type=application/json";
83   
 
84  1 toggle static
85    {
86  1 domainData = new HashMap<>();
87  1 domainData.put(DEFAULT_ENSEMBL_BASEURL,
88    new EnsemblData(DEFAULT_ENSEMBL_BASEURL, LATEST_ENSEMBL_REST_VERSION));
89  1 domainData.put(DEFAULT_ENSEMBL_GENOMES_BASEURL, new EnsemblData(
90    DEFAULT_ENSEMBL_GENOMES_BASEURL, LATEST_ENSEMBLGENOMES_REST_VERSION));
91    }
92   
93    protected volatile boolean inProgress = false;
94   
95    /**
96    * Default constructor to use rest.ensembl.org
97    */
 
98  36 toggle public EnsemblRestClient()
99    {
100  36 super();
101   
102    /*
103    * initialise domain info lazily
104    */
105  36 if (!domainData.containsKey(ensemblDomain))
106    {
107  0 domainData.put(ensemblDomain,
108    new EnsemblData(ensemblDomain, LATEST_ENSEMBL_REST_VERSION));
109    }
110  36 if (!domainData.containsKey(ensemblGenomesDomain))
111    {
112  0 domainData.put(ensemblGenomesDomain, new EnsemblData(
113    ensemblGenomesDomain, LATEST_ENSEMBLGENOMES_REST_VERSION));
114    }
115    }
116   
117    /**
118    * Constructor given the target domain to fetch data from
119    *
120    * @param d
121    */
 
122  1 toggle public EnsemblRestClient(String d)
123    {
124  1 setDomain(d);
125    }
126   
 
127  0 toggle @Override
128    public boolean queryInProgress()
129    {
130  0 return inProgress;
131    }
132   
 
133  0 toggle @Override
134    public StringBuffer getRawRecords()
135    {
136  0 return null;
137    }
138   
139    /**
140    * Returns the URL for the client http request
141    *
142    * @param ids
143    * @return
144    * @throws MalformedURLException
145    */
146    protected abstract URL getUrl(List<String> ids)
147    throws MalformedURLException;
148   
149    /**
150    * Returns true if client uses GET method, false if it uses POST
151    *
152    * @return
153    */
154    protected abstract boolean useGetRequest();
155   
156    /**
157    * Return the desired value for the Content-Type request header
158    *
159    * @param multipleIds
160    *
161    * @return
162    * @see https://github.com/Ensembl/ensembl-rest/wiki/HTTP-Headers
163    */
164    protected abstract String getRequestMimeType(boolean multipleIds);
165   
166    /**
167    * Return the desired value for the Accept request header
168    *
169    * @return
170    * @see https://github.com/Ensembl/ensembl-rest/wiki/HTTP-Headers
171    */
172    protected abstract String getResponseMimeType();
173   
174    /**
175    * Checks Ensembl's REST 'ping' endpoint, and returns true if response
176    * indicates available, else false
177    *
178    * @see http://rest.ensembl.org/documentation/info/ping
179    * @return
180    */
 
181  0 toggle boolean checkEnsembl()
182    {
183  0 BufferedReader br = null;
184  0 String pingUrl = getDomain() + "/info/ping" + CONTENT_TYPE_JSON;
185  0 try
186    {
187    // note this format works for both ensembl and ensemblgenomes
188    // info/ping.json works for ensembl only (March 2016)
189  0 URL ping = new URL(pingUrl);
190   
191    /*
192    * expect {"ping":1} if ok
193    * if ping takes more than 2 seconds to respond, treat as if unavailable
194    */
195  0 br = getHttpResponse(ping, null, 2 * 1000);
196  0 if (br == null)
197    {
198    // error reponse status
199  0 return false;
200    }
201  0 JSONParser jp = new JSONParser();
202  0 JSONObject val = (JSONObject) jp.parse(br);
203  0 String pingString = val.get("ping").toString();
204  0 return pingString != null;
205    } catch (Throwable t)
206    {
207  0 System.err.println(
208    "Error connecting to " + pingUrl + ": " + t.getMessage());
209    } finally
210    {
211  0 if (br != null)
212    {
213  0 try
214    {
215  0 br.close();
216    } catch (IOException e)
217    {
218    // ignore
219    }
220    }
221    }
222  0 return false;
223    }
224   
225    /**
226    * returns a reader to a Fasta response from the Ensembl sequence endpoint
227    *
228    * @param ids
229    * @return
230    * @throws IOException
231    */
 
232  0 toggle protected FileParse getSequenceReader(List<String> ids) throws IOException
233    {
234  0 URL url = getUrl(ids);
235   
236  0 BufferedReader reader = getHttpResponse(url, ids);
237  0 if (reader == null)
238    {
239    // request failed
240  0 return null;
241    }
242  0 FileParse fp = new FileParse(reader, url.toString(),
243    DataSourceType.URL);
244  0 return fp;
245    }
246   
247    /**
248    * Gets a reader to the HTTP response, using the default read timeout of 5
249    * minutes
250    *
251    * @param url
252    * @param ids
253    * @return
254    * @throws IOException
255    */
 
256  0 toggle protected BufferedReader getHttpResponse(URL url, List<String> ids)
257    throws IOException
258    {
259  0 return getHttpResponse(url, ids, DEFAULT_READ_TIMEOUT);
260    }
261   
262    /**
263    * Sends the HTTP request and gets the response as a reader. Returns null if
264    * the HTTP response code was not 200.
265    *
266    * @param url
267    * @param ids
268    * written as Json POST body if more than one
269    * @param readTimeout
270    * in milliseconds
271    * @return
272    * @throws IOException
273    */
 
274  0 toggle protected BufferedReader getHttpResponse(URL url, List<String> ids,
275    int readTimeout) throws IOException
276    {
277  0 int retriesLeft = MAX_RETRIES;
278  0 HttpURLConnection connection = null;
279  0 int responseCode = 0;
280   
281  0 while (retriesLeft > 0)
282    {
283  0 connection = tryConnection(url, ids, readTimeout);
284  0 responseCode = connection.getResponseCode();
285  0 if (responseCode == HTTP_OVERLOAD) // 429
286    {
287  0 retriesLeft--;
288  0 checkRetryAfter(connection);
289    }
290    else
291    {
292  0 retriesLeft = 0;
293    }
294    }
295  0 if (responseCode != HTTP_OK) // 200
296    {
297    /*
298    * note: a GET request for an invalid id returns an error code e.g. 415
299    * but POST request returns 200 and an empty Fasta response
300    */
301  0 System.err.println("Response code " + responseCode + " for " + url);
302  0 return null;
303    }
304   
305  0 InputStream response = connection.getInputStream();
306   
307    // System.out.println(getClass().getName() + " took "
308    // + (System.currentTimeMillis() - now) + "ms to fetch");
309   
310  0 BufferedReader reader = null;
311  0 reader = new BufferedReader(new InputStreamReader(response, "UTF-8"));
312  0 return reader;
313    }
314   
315    /**
316    * @param url
317    * @param ids
318    * @param readTimeout
319    * @return
320    * @throws IOException
321    * @throws ProtocolException
322    */
 
323  0 toggle protected HttpURLConnection tryConnection(URL url, List<String> ids,
324    int readTimeout) throws IOException, ProtocolException
325    {
326    // System.out.println(System.currentTimeMillis() + " " + url);
327  0 HttpURLConnection connection = (HttpURLConnection) url.openConnection();
328   
329    /*
330    * POST method allows multiple queries in one request; it is supported for
331    * sequence queries, but not for overlap
332    */
333  0 boolean multipleIds = ids != null && ids.size() > 1;
334  0 connection.setRequestMethod(
335  0 multipleIds ? HttpMethod.POST : HttpMethod.GET);
336  0 connection.setRequestProperty("Content-Type",
337    getRequestMimeType(multipleIds));
338  0 connection.setRequestProperty("Accept", getResponseMimeType());
339   
340  0 connection.setDoInput(true);
341  0 connection.setDoOutput(multipleIds);
342   
343  0 if (!Jalview.isJS())
344    {
345  0 connection.setUseCaches(false);
346  0 connection.setConnectTimeout(CONNECT_TIMEOUT_MS);
347  0 connection.setReadTimeout(readTimeout);
348    }
349   
350  0 if (multipleIds)
351    {
352  0 writePostBody(connection, ids);
353    }
354  0 return connection;
355    }
356   
357    /**
358    * Inspects response headers for a 'retry-after' directive, and waits for the
359    * directed period (if less than 10 seconds)
360    *
361    * @see https://github.com/Ensembl/ensembl-rest/wiki/Rate-Limits
362    * @param connection
363    */
 
364  0 toggle void checkRetryAfter(HttpURLConnection connection)
365    {
366  0 String retryDelay = connection.getHeaderField("Retry-After");
367   
368    // to test:
369    // retryDelay = "5";
370   
371  0 if (retryDelay != null)
372    {
373  0 try
374    {
375  0 int retrySecs = Integer.valueOf(retryDelay);
376  0 if (retrySecs > 0 && retrySecs < 10)
377    {
378  0 System.err
379    .println("Ensembl REST service rate limit exceeded, waiting "
380    + retryDelay + " seconds before retrying");
381  0 Thread.sleep(1000 * retrySecs);
382    }
383    } catch (NumberFormatException | InterruptedException e)
384    {
385  0 System.err.println("Error handling Retry-After: " + e.getMessage());
386    }
387    }
388    }
389   
390    /**
391    * Rechecks if Ensembl is responding, unless the last check was successful and
392    * the retest interval has not yet elapsed. Returns true if Ensembl is up,
393    * else false. Also retrieves and saves the current version of Ensembl data
394    * and REST services at intervals.
395    *
396    * @return
397    */
 
398  0 toggle protected boolean isEnsemblAvailable()
399    {
400  0 EnsemblData info = domainData.get(getDomain());
401   
402  0 long now = System.currentTimeMillis();
403   
404    /*
405    * recheck if Ensembl is up if it was down, or the recheck period has elapsed
406    */
407  0 boolean retestAvailability = (now
408    - info.lastAvailableCheckTime) > AVAILABILITY_RETEST_INTERVAL;
409  0 if (!info.restAvailable || retestAvailability)
410    {
411  0 info.restAvailable = checkEnsembl();
412  0 info.lastAvailableCheckTime = now;
413    }
414   
415    /*
416    * refetch Ensembl versions if the recheck period has elapsed
417    */
418  0 boolean refetchVersion = (now
419    - info.lastVersionCheckTime) > VERSION_RETEST_INTERVAL;
420  0 if (refetchVersion)
421    {
422  0 checkEnsemblRestVersion();
423  0 checkEnsemblDataVersion();
424  0 info.lastVersionCheckTime = now;
425    }
426   
427  0 return info.restAvailable;
428    }
429   
430    /**
431    * Constructs, writes and flushes the POST body of the request, containing the
432    * query ids in JSON format
433    *
434    * @param connection
435    * @param ids
436    * @throws IOException
437    */
 
438  0 toggle protected void writePostBody(HttpURLConnection connection,
439    List<String> ids) throws IOException
440    {
441  0 boolean first;
442  0 StringBuilder postBody = new StringBuilder(64);
443  0 postBody.append("{\"ids\":[");
444  0 first = true;
445  0 for (String id : ids)
446    {
447  0 if (!first)
448    {
449  0 postBody.append(",");
450    }
451  0 first = false;
452  0 postBody.append("\"");
453  0 postBody.append(id.trim());
454  0 postBody.append("\"");
455    }
456  0 postBody.append("]}");
457  0 byte[] thepostbody = postBody.toString().getBytes();
458  0 connection.setRequestProperty("Content-Length",
459    Integer.toString(thepostbody.length));
460  0 DataOutputStream wr = new DataOutputStream(
461    connection.getOutputStream());
462  0 wr.write(thepostbody);
463  0 wr.flush();
464  0 wr.close();
465    }
466   
467    /**
468    * Fetches and checks Ensembl's REST version number
469    *
470    * @return
471    */
 
472  0 toggle private void checkEnsemblRestVersion()
473    {
474  0 EnsemblData info = domainData.get(getDomain());
475   
476  0 JSONParser jp = new JSONParser();
477  0 URL url = null;
478  0 try
479    {
480  0 url = new URL(getDomain() + "/info/rest" + CONTENT_TYPE_JSON);
481  0 BufferedReader br = getHttpResponse(url, null);
482  0 if (br == null)
483    {
484  0 return;
485    }
486  0 JSONObject val = (JSONObject) jp.parse(br);
487  0 String version = val.get("release").toString();
488  0 String majorVersion = version.substring(0, version.indexOf("."));
489  0 String expected = info.expectedRestVersion;
490  0 String expectedMajorVersion = expected.substring(0,
491    expected.indexOf("."));
492  0 info.restMajorVersionMismatch = false;
493  0 try
494    {
495    /*
496    * if actual REST major version is ahead of what we expect,
497    * record this in case we want to warn the user
498    */
499  0 if (Float.valueOf(majorVersion) > Float
500    .valueOf(expectedMajorVersion))
501    {
502  0 info.restMajorVersionMismatch = true;
503    }
504    } catch (NumberFormatException e)
505    {
506  0 System.err.println("Error in REST version: " + e.toString());
507    }
508   
509    /*
510    * check if REST version is later than what Jalview has tested against,
511    * if so warn; we don't worry if it is earlier (this indicates Jalview has
512    * been tested in advance against the next pending REST version)
513    */
514  0 boolean laterVersion = StringUtils.compareVersions(version,
515    expected) == 1;
516  0 if (laterVersion)
517    {
518  0 System.err.println(String.format(
519    "EnsemblRestClient expected %s REST version %s but found %s, see %s",
520    getDbSource(), expected, version, REST_CHANGE_LOG));
521    }
522  0 info.restVersion = version;
523    } catch (Throwable t)
524    {
525  0 System.err.println(
526    "Error checking Ensembl REST version: " + t.getMessage());
527    }
528    }
529   
 
530  0 toggle public boolean isRestMajorVersionMismatch()
531    {
532  0 return domainData.get(getDomain()).restMajorVersionMismatch;
533    }
534   
535    /**
536    * Fetches and checks Ensembl's data version number
537    *
538    * @return
539    */
 
540  0 toggle private void checkEnsemblDataVersion()
541    {
542  0 JSONParser jp = new JSONParser();
543  0 URL url = null;
544  0 BufferedReader br = null;
545   
546  0 try
547    {
548  0 url = new URL(getDomain() + "/info/data" + CONTENT_TYPE_JSON);
549  0 br = getHttpResponse(url, null);
550  0 if (br != null)
551    {
552  0 JSONObject val = (JSONObject) jp.parse(br);
553  0 JSONArray versions = (JSONArray) val.get("releases");
554  0 domainData.get(getDomain()).dataVersion = versions.get(0)
555    .toString();
556    }
557    } catch (Throwable t)
558    {
559  0 System.err.println(
560    "Error checking Ensembl data version: " + t.getMessage());
561    } finally
562    {
563  0 if (br != null)
564    {
565  0 try
566    {
567  0 br.close();
568    } catch (IOException e)
569    {
570    // ignore
571    }
572    }
573    }
574    }
575   
 
576  0 toggle public String getEnsemblDataVersion()
577    {
578  0 return domainData.get(getDomain()).dataVersion;
579    }
580   
 
581  0 toggle @Override
582    public String getDbVersion()
583    {
584  0 return getEnsemblDataVersion();
585    }
586   
587    }
  • Report generated by OpenClover v 4.2.1 on Wed Oct 31 2018 15:15:54 GMT using coverage data from Wed Oct 31 2018 15:15:47 GMT.
  • Clover free edition.