Skip to content

Commit ecd9c8c

Browse files
dr0idr0i
committed
WIP SRUopener (#510)
1 parent b9eebb4 commit ecd9c8c

File tree

3 files changed

+267
-0
lines changed

3 files changed

+267
-0
lines changed

‎metafacture-biblio/src/main/java/org/metafacture/biblio/SruOpener.java‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
/* Copyright 2013 Pascal Christoph.
2+
* Licensed under the Eclipse Public License 1.0 */
3+
4+
package org.metafacture.biblio;
5+
6+
import org.metafacture.framework.FluxCommand;
7+
import org.metafacture.framework.MetafactureException;
8+
import org.metafacture.framework.ObjectReceiver;
9+
import org.metafacture.framework.annotations.Description;
10+
import org.metafacture.framework.annotations.In;
11+
import org.metafacture.framework.annotations.Out;
12+
import org.metafacture.framework.helpers.DefaultObjectPipe;
13+
14+
import java.io.IOException;
15+
import java.io.InputStream;
16+
import java.io.InputStreamReader;
17+
import java.io.Reader;
18+
import java.net.HttpURLConnection;
19+
import java.net.URL;
20+
21+
/**
22+
* Opens an SRU (Search Retrieval by URL) stream and passes a reader to the receiver.
23+
* The input should be the base URL of the SRU service to be retrieved from.
24+
*
25+
* @author Pascal Christoph (dr0i)
26+
*/
27+
@Description("Opens a SRU stream and passes a reader to the receiver. The input should be the base URL of the SRU service to be retrieved from. Mandatory argument is: QUERY.")
28+
@In(String.class)
29+
@Out(Reader.class)
30+
@FluxCommand("open-sru")
31+
public final class SruOpener extends DefaultObjectPipe<String, ObjectReceiver<Reader>> {
32+
33+
private static final String OPERATION = "searchRetrieve";
34+
private static final String RECORD_SCHEMA = "MARC21-xml";
35+
private static final String USER_AGENT = "";
36+
private static final String VERSION = "2.0";
37+
38+
private static final int CONNECTION_TIMEOUT = 11000;
39+
private static final int MAXIMUM_RECORDS = 10;
40+
private static final int START_RECORD = 1;
41+
private static final int MINIMUM_LOOKUP_LENGTH =768;
42+
43+
private String operation = OPERATION;
44+
private String query;
45+
private String recordSchema = RECORD_SCHEMA;
46+
private String userAgent = USER_AGENT;
47+
private String version = VERSION;
48+
49+
private int maximumRecords = MAXIMUM_RECORDS;
50+
private int startRecord = START_RECORD;
51+
private int totalRecords;
52+
53+
private boolean stopRetrieving;
54+
55+
56+
/**
57+
* Creates an instance of {@link SruOpener}
58+
*/
59+
public SruOpener() {
60+
}
61+
62+
/**
63+
* Sets the User Agent to use. <strong>Default value: {@value USER_AGENT}</strong>.
64+
*
65+
* @param userAgent a user agent to be used when opening a URL
66+
*/
67+
public void setUserAgent(final String userAgent) {
68+
this.userAgent = userAgent;
69+
}
70+
71+
/**
72+
* Sets the query of the search.
73+
* <strong>Setting a query is mandatory.</strong>
74+
*
75+
* @param query the query
76+
*/
77+
78+
public void setQuery(final String query) {
79+
this.query = query;
80+
}
81+
82+
/**
83+
* Sets total number of records to be retrieved. <strong>Default value: indefinite (as in "all")</strong>.
84+
*
85+
* @param totalRecords total number of records to be retrieved
86+
*/
87+
public void setTotal(final String totalRecords) {
88+
this.totalRecords = Integer.parseInt(totalRecords);
89+
}
90+
91+
/**
92+
* Sets the maximum of records returned in one lookup. <strong>Default value: {@value MAXIMUM_RECORDS}</strong>.
93+
* The lookup is repeated as long as {@link #maximumRecords} is lesser than {@link #totalRecords}.
94+
*
95+
* @param maximumRecords maximum of records returned in one lookup
96+
*/
97+
public void setMaximumRecords(final String maximumRecords) {
98+
this.maximumRecords = Integer.parseInt(maximumRecords);
99+
}
100+
101+
/**
102+
* Sets where to start when retrieving records. <strong>Default value: {@value START_RECORD}</strong>.
103+
*
104+
* @param startRecord where to start when retrieving records
105+
*/
106+
public void setStartRecord(final String startRecord) {
107+
this.startRecord = Integer.parseInt(startRecord);
108+
}
109+
110+
/**
111+
* Sets the format of the retrieved record data. <strong>Default value: {@value RECORD_SCHEMA}</strong>.
112+
*
113+
* @param recordSchema the format of the data of the records
114+
*/
115+
public void setRecordSchema(final String recordSchema) {
116+
this.recordSchema = recordSchema;
117+
}
118+
119+
/**
120+
* Sets the kind of operation of the lookup. <strong>Default value: {@value OPERATION}</strong>.
121+
*
122+
* @param operation the kind of operation of the lookup
123+
*/
124+
public void setOperation(final String operation) {
125+
this.operation = operation;
126+
}
127+
128+
/**
129+
* Sets the version of the lookup. <strong>Default value: {@value VERSION}</strong>.
130+
*
131+
* @param version the version of the lookup
132+
*/
133+
public void setVersion(final String version) {
134+
this.version = version;
135+
}
136+
137+
@Override
138+
public void process(final String baseUrl) {
139+
140+
try {
141+
142+
StringBuilder srUrl = new StringBuilder(baseUrl);
143+
if (query != null) {
144+
srUrl.append("?query=").append(query).append("&operation=").append(operation).append("&recordSchema=").append(recordSchema).append("&version=").append(version);
145+
}
146+
else {
147+
throw new IllegalArgumentException("Missing mandatory parameter 'query'");
148+
}
149+
int retrievedRecords = 0;
150+
while (!stopRetrieving && (totalRecords==0 || retrievedRecords < totalRecords)) {
151+
if (totalRecords >0) {
152+
int yetToRetrieveRecords = totalRecords - retrievedRecords ;
153+
if (yetToRetrieveRecords < maximumRecords) {
154+
maximumRecords = yetToRetrieveRecords;
155+
}
156+
}
157+
retrieve(srUrl, startRecord, maximumRecords); //todo: bis max lookup zuviel (bis der nämlich sehr klein ist => keine
158+
// Ergebnisse mehr)
159+
startRecord = startRecord + maximumRecords;
160+
retrievedRecords = retrievedRecords + maximumRecords;
161+
}
162+
}
163+
catch (final IOException e) {
164+
throw new MetafactureException(e);
165+
}
166+
}
167+
168+
private void retrieve(StringBuilder srUrl, int startRecord, int maximumRecords) throws IOException {
169+
final URL urlToOpen = new URL(srUrl.toString() + "&maximumRecords=" + maximumRecords+"&startRecord=" + startRecord);
170+
final HttpURLConnection connection = (HttpURLConnection) urlToOpen.openConnection();
171+
172+
connection.setConnectTimeout(CONNECTION_TIMEOUT);
173+
if (!userAgent.isEmpty()) {
174+
connection.setRequestProperty("User-Agent", userAgent);
175+
}
176+
InputStream istream = getInputStream(connection);
177+
try (
178+
InputStreamReader inputStreamReader = new InputStreamReader(istream);
179+
) {
180+
System.out.println("urlToOpen="+urlToOpen.toString());
181+
System.out.print(", startRecord="+startRecord);
182+
System.out.print(", maximumRecords="+maximumRecords);
183+
System.out.print(", istream.length="+istream.available());
184+
System.out.println();
185+
if (istream.available() < 768){ // we take it that this is a result without a record
186+
stopRetrieving = true;
187+
}
188+
189+
getReceiver().process(inputStreamReader);
190+
}
191+
}
192+
193+
private InputStream getInputStream(final HttpURLConnection connection) {
194+
try {
195+
return connection.getInputStream();
196+
}
197+
catch (final IOException e) {
198+
stopRetrieving = true;
199+
return connection.getErrorStream();
200+
}
201+
}
202+
203+
}

‎metafacture-biblio/src/main/resources/flux-commands.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,4 @@ decode-aseq org.metafacture.biblio.AseqDecoder
2929
decode-mab org.metafacture.biblio.MabDecoder
3030

3131
open-oaipmh org.metafacture.biblio.OaiPmhOpener
32+
open-sru org.metafacture.biblio.SruOpener

‎metafacture-biblio/src/test/java/org/metafacture/biblio/SruOpenerTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package org.metafacture.biblio;
2+
3+
import org.junit.Test;
4+
import org.metafacture.framework.ObjectReceiver;
5+
6+
import java.io.BufferedReader;
7+
import java.io.IOException;
8+
import java.io.Reader;
9+
10+
public class SruOpenerTest {
11+
12+
private StringBuilder resultCollector = new StringBuilder();
13+
private int resultCollectorsResetStreamCount;
14+
15+
16+
// @Test
17+
public void test(){
18+
SruOpener sruOpener = new SruOpener();
19+
sruOpener.setReceiver(new ObjectReceiver<Reader> () {
20+
21+
@Override
22+
public void process(final Reader obj) {
23+
BufferedReader in = new BufferedReader(obj);
24+
String line = null;
25+
StringBuilder rslt = new StringBuilder();
26+
while (true) {
27+
try {
28+
if (!((line = in.readLine()) != null)) break;
29+
}
30+
catch (IOException e) {
31+
throw new RuntimeException(e);
32+
}
33+
rslt.append(line);
34+
}
35+
String result = rslt.toString();
36+
if (result.length() > 768) {
37+
System.out.println(rslt.toString().substring(768, 1024));
38+
}
39+
else System.out.println("Antwort zu klein, gehe von ende der Anzhal der Records aus");
40+
resultCollector.append(obj);
41+
}
42+
43+
@Override
44+
public void resetStream() {
45+
++resultCollectorsResetStreamCount;
46+
}
47+
48+
@Override
49+
public void closeStream() {
50+
51+
}
52+
});
53+
54+
// sruOpener.setQuery("dnb.isil%3DDE-Sol1");
55+
sruOpener.setQuery("WVN%3D24A05");
56+
sruOpener.setRecordSchema("MARC21plus-xml");
57+
sruOpener.setVersion("1.1");
58+
sruOpener.setStartRecord("1890");
59+
sruOpener.setTotal("32");
60+
sruOpener.process("https://services.dnb.de/sru/dnb");
61+
// System.out.println(resultCollector.toString());
62+
}
63+
}

0 commit comments

Comments
 (0)