Package com.marklogic.contentpump
Class RDFReader<VALUEIN>
- java.lang.Object
-
- org.apache.hadoop.mapreduce.RecordReader<DocumentURIWithSourceInfo,VALUEIN>
-
- com.marklogic.contentpump.ImportRecordReader<VALUEIN>
-
- com.marklogic.contentpump.RDFReader<VALUEIN>
-
- Type Parameters:
VALUEIN-
- All Implemented Interfaces:
ConfigConstants,Closeable,AutoCloseable
- Direct Known Subclasses:
CompressedRDFReader
public class RDFReader<VALUEIN> extends ImportRecordReader<VALUEIN>
Reader for RDF quads/triples. Uses Jena library to parse RDF and sends triples to the database in groups of MAXTRIPLESPERDOCUMENT.
-
-
Nested Class Summary
Nested Classes Modifier and Type Class Description protected classRDFReader.ParserErrorHandlerprotected classRDFReader.RunnableParser
-
Field Summary
Fields Modifier and Type Field Description protected StringBuilderbufferprotected Stringcollectionprotected intcollectionCountprotected Hashtable<String,Vector>collectionHashprotected booleancompressedprotected intcountPerBatchprotected org.apache.jena.query.Datasetdatasetstatic StringDEFAULT_GRAPHprotected com.marklogic.xcc.ContentPermission[]defaultPermsprotected longendprotected HashMap<String,com.marklogic.xcc.ContentPermission[]>existingMapPermsprotected Iterator<String>graphItrprotected Iterator<String>graphNameIterprotected StringBuildergraphQryprotected booleangraphSupportedstatic StringHASHALGORITHMprotected booleanhasNextprotected booleanhasOutputColprotected IdGeneratoridGenprotected booleanignoreCollectionQuadprotected longingestedTriplesprotected longINGESTIONNOTIFYSTEPprotected longINMEMORYTHRESHOLDprotected StringinputFnstatic StringJENA_DEFAULT_GRAPHprotected RDFReader.RunnableParserjenaStreamingParserprotected org.apache.jena.riot.Langlangstatic org.apache.commons.logging.LogLOGprotected intMAXGRAPHSPERREQUESTprotected intMAXTRIPLESPERDOCUMENTprotected longmilliSecsprotected HashSet<String>newGraphsprotected StringorigFnprotected StringoutputGraphprotected StringoutputOverrideGraphprotected static Pattern[]patternsprotected longposprotected Randomrandomprotected longrandomValueprotected StreamrdfInputStreamprotected IteratorrdfIterprotected LinkedMapWritableroleMapprotected booleanroleMapExistsprotected longsplitStartprotected longstartprotected org.apache.jena.rdf.model.StmtIteratorstatementIterprotected Stringversion-
Fields inherited from class com.marklogic.contentpump.ImportRecordReader
conf, encoding, file, fs, iterator, key, mode, streaming, subId, value
-
Fields inherited from interface com.marklogic.contentpump.ConfigConstants
AGGREGATE_RECORD_ELEMENT, AGGREGATE_RECORD_NAMESPACE, AGGREGATE_URI_ID, API_KEY, ARCHIVE_METADATA_OPTIONAL, AUDIT_MLCPFINISH_CODE, AUDIT_MLCPFINISH_EVENT, AUDIT_MLCPSTART_CODE, AUDIT_MLCPSTART_EVENT, BASE_PATH, BATCH_MIN_VERSION, BATCH_SIZE, COLLECTION_FILTER, CONF_AGGREGATE_RECORD_ELEMENT, CONF_AGGREGATE_RECORD_NAMESPACE, CONF_AUDIT_MLCPFINISH_ENABLED, CONF_AUDIT_MLCPFINISH_MESSAGE, CONF_AUDIT_MLCPSTART_MESSAGE, CONF_COPY_PERMISSIONS, CONF_COPY_PROPERTIES, CONF_DELIMITED_DATA_TYPE, CONF_DELIMITED_ROOT_NAME, CONF_DELIMITER, CONF_INPUT_ARCHIVE_METADATA_OPTIONAL, CONF_INPUT_COMPRESSION_CODEC, CONF_INPUT_DIRECTORY, CONF_INPUT_FILE_PATTERN, CONF_INPUT_GENERATE_URI, CONF_INPUT_MODULES_DATABASE, CONF_INPUT_MODULES_ROOT, CONF_INPUT_PATH_FILTER_CLASS, CONF_INPUT_SEQUENCEFILE_KEY_CLASS, CONF_INPUT_SEQUENCEFILE_VALUE_CLASS, CONF_INPUT_SEQUENCEFILE_VALUE_TYPE, CONF_INPUT_URI_ID, CONF_MAPREDUCE_JOB_MAP_CLASS, CONF_MAPREDUCE_JOB_WORKING_DIR, CONF_MAPREDUCE_JOBTRACKER_ADDRESS, CONF_MAX_SPLIT_SIZE1, CONF_MAX_SPLIT_SIZE2, CONF_MIMETYPES, CONF_MIN_SPLIT_SIZE1, CONF_MIN_SPLIT_SIZE2, CONF_MIN_THREADS, CONF_ML_VERSION, CONF_MULTITHREADEDMAPPER_CLASS, CONF_OUTPUT_FILENAME_AS_COLLECTION, CONF_OUTPUT_FILEPATH, CONF_OUTPUT_TYPE, CONF_ROLE_MAP, CONF_SPLIT_INPUT, CONF_THREADS_PER_SPLIT, CONF_TRANSFORM_FUNCTION, CONF_TRANSFORM_MODULE, CONF_TRANSFORM_NAMESPACE, CONF_TRANSFORM_PARAM, CONTENT_ENCODING, CONTENTPUMP_BUNDLE_ARTIFACT, CONTENTPUMP_HOME_PROPERTY_NAME, CONTENTPUMP_JAR_PREFIX, COPY_COLLECTIONS, COPY_METADATA, COPY_PERMISSIONS, COPY_PROPERTIES, COPY_QUALITY, DATA_TYPE, DATABASE, DEFAULT_ARCHIVE_METADATA_OPTIONAL, DEFAULT_COPY_COLLECTIONS, DEFAULT_COPY_METADATA, DEFAULT_COPY_PERMISSIONS, DEFAULT_COPY_PROPERTIES, DEFAULT_COPY_QUALITY, DEFAULT_DELIMITER, DEFAULT_ENCODING, DEFAULT_ML_CLOUD_PORT, DEFAULT_OUTPUT_TYPE, DEFAULT_PORT, DEFAULT_SEQUENCEFILE_VALUE_TYPE, DEFAULT_THREAD_COUNT, DELIMITED_ROOT_NAME, DELIMITED_URI_ID, DELIMITER, DIRECTORY_FILTER, DOCUMENT_SELECTOR, DOCUMENT_TYPE, FAST_LOAD, GENERATE_URI, GRANT_TYPE, HADOOP_CONF_DIR, HADOOP_CONFDIR_ENV_NAME, HOST, INPUT_API_KEY, INPUT_BASE_PATH, INPUT_COMPRESSED, INPUT_COMPRESSION_CODEC, INPUT_DATABASE, INPUT_FILE_PATH, INPUT_FILE_PATTERN, INPUT_FILE_TYPE, INPUT_FILE_TYPE_DEFAULT, INPUT_GRANT_TYPE, INPUT_HOST, INPUT_KEYSTORE_PASSWD, INPUT_KEYSTORE_PATH, INPUT_PASSWORD, INPUT_PORT, INPUT_SEQUENCEFILE_KEY_CLASS, INPUT_SEQUENCEFILE_VALUE_CLASS, INPUT_SEQUENCEFILE_VALUE_TYPE, INPUT_SSL, INPUT_SSL_PROTOCOL, INPUT_TOKEN_DURATION, INPUT_TOKEN_ENDPOINT, INPUT_TRUSTSTORE_PASSWD, INPUT_TRUSTSTORE_PATH, INPUT_USERNAME, KEYSTORE_PASSWD, KEYSTORE_PATH, MAX_BATCH_SIZE, MAX_SPLIT_SIZE, MAX_THREAD_PERCENTAGE, MAX_THREADS, MAX_TXN_SIZE, MIN_SPLIT_SIZE, MODE, MODULES, MODULES_ROOT, NAMESPACE, OPTIONS_FILE, OUTPUT_API_KEY, OUTPUT_BASE_PATH, OUTPUT_CLEANDIR, OUTPUT_COLLECTIONS, OUTPUT_COMPRESS, OUTPUT_DATABASE, OUTPUT_DIRECTORY, OUTPUT_FILE_PATH, OUTPUT_FILENAME_AS_COLLECTION, OUTPUT_GRANT_TYPE, OUTPUT_GRAPH, OUTPUT_HOST, OUTPUT_IDNAME, OUTPUT_INDENTED, OUTPUT_KEYSTORE_PASSWD, OUTPUT_KEYSTORE_PATH, OUTPUT_LANGUAGE, OUTPUT_OVERRIDE_GRAPH, OUTPUT_PARTITION, OUTPUT_PASSWORD, OUTPUT_PERMISSIONS, OUTPUT_PORT, OUTPUT_QUALITY, OUTPUT_SSL, OUTPUT_SSL_PROTOCOL, OUTPUT_TOKEN_DURATION, OUTPUT_TOKEN_ENDPOINT, OUTPUT_TRUSTSTORE_PASSWD, OUTPUT_TRUSTSTORE_PATH, OUTPUT_TYPE, OUTPUT_URI_PREFIX, OUTPUT_URI_REPLACE, OUTPUT_URI_SUFFIX, OUTPUT_USERNAME, PASSWORD, PATH_NAMESPACE, POLLING_INIT_DELAY, POLLING_PERIOD, POLLING_TIME_UNIT, PORT, QUERY_FILTER, RDF_STREAMING_MEMORY_THRESHOLD, RDF_TRIPLES_PER_DOCUMENT, REDACTION, RESTRICT_HOSTS, RESTRICT_INPUT_HOSTS, RESTRICT_OUTPUT_HOSTS, SNAPSHOT, SPLIT_INPUT, SSL, SSL_PROTOCOL, STREAMING, TEMPORAL_COLLECTION, THREAD_COUNT, THREADS_PER_SPLIT, TOKEN_DURATION, TOKEN_ENDPOINT, TRANSACTION_SIZE, TRANSFORM_FUNCTION, TRANSFORM_MODULE, TRANSFORM_NAMESPACE, TRANSFORM_PARAM, TRUSTSTORE_PASSWD, TRUSTSTORE_PATH, TYPE_FILTER, URI_ID, USERNAME, XML_REPAIR_LEVEL
-
-
Constructor Summary
Constructors Constructor Description RDFReader(String version, LinkedMapWritable roleMap)
-
Method Summary
All Methods Static Methods Instance Methods Concrete Methods Modifier and Type Method Description voidclose()protected static StringescapeXml(String _in)floatgetProgress()voidinitExistingMapPerms()voidinitialize(org.apache.hadoop.mapreduce.InputSplit inSplit, org.apache.hadoop.mapreduce.TaskAttemptContext context)protected voidinitParser(String fsname, long size)protected voidinitStream(org.apache.hadoop.mapreduce.InputSplit inSplit)com.marklogic.xcc.ContentPermission[]insertGraphDoc(String graph)protected StringlargestCollection()protected voidloadModel(String fsname, InputStream in)booleannextInMemoryKeyValue()booleannextInMemoryQuadKeyValue()booleannextInMemoryQuadKeyValueIgnoreCollections()booleannextInMemoryQuadKeyValueWithCollections()booleannextInMemoryTripleKeyValue()booleannextKeyValue()booleannextStramingQuadKeyValue()booleannextStreamingKeyValue()protected booleannextStreamingQuadKeyValueIgnoreCollections()booleannextStreamingQuadKeyValueWithCollections()protected booleannextStreamingTripleKeyValue()protected voidnotifyUser()protected Stringobject(org.apache.jena.graph.Node node)protected voidparse(String fsname, org.apache.hadoop.fs.FSDataInputStream in)protected Stringpredicate(org.apache.jena.graph.Node subj)protected Stringpredicate(org.apache.jena.rdf.model.Resource subj)protected Stringresource(org.apache.jena.graph.Node rsrc)protected Stringresource(org.apache.jena.graph.Node rsrc, String tag)protected Stringresource(org.apache.jena.rdf.model.Resource rsrc, String tag)protected voidsetKey()protected Stringsubject(org.apache.jena.graph.Node subj)protected Stringsubject(org.apache.jena.rdf.model.Resource subj)protected voidsubmitGraphQuery()protected voidwrite(String str)voidwriteValue()voidwriteValue(String collection)-
Methods inherited from class com.marklogic.contentpump.ImportRecordReader
configFileNameAsCollection, getCurrentKey, getCurrentValue, getFile, initConfig, makeURIForZipEntry, makeURIFromPath, openFile, setFile, setKey, setSkipKey
-
-
-
-
Field Detail
-
LOG
public static final org.apache.commons.logging.Log LOG
-
HASHALGORITHM
public static final String HASHALGORITHM
- See Also:
- Constant Field Values
-
DEFAULT_GRAPH
public static final String DEFAULT_GRAPH
- See Also:
- Constant Field Values
-
JENA_DEFAULT_GRAPH
public static final String JENA_DEFAULT_GRAPH
- See Also:
- Constant Field Values
-
patterns
protected static Pattern[] patterns
-
MAXTRIPLESPERDOCUMENT
protected int MAXTRIPLESPERDOCUMENT
-
MAXGRAPHSPERREQUEST
protected int MAXGRAPHSPERREQUEST
-
countPerBatch
protected int countPerBatch
-
INMEMORYTHRESHOLD
protected long INMEMORYTHRESHOLD
-
INGESTIONNOTIFYSTEP
protected long INGESTIONNOTIFYSTEP
-
dataset
protected org.apache.jena.query.Dataset dataset
-
statementIter
protected org.apache.jena.rdf.model.StmtIterator statementIter
-
collection
protected String collection
-
jenaStreamingParser
protected RDFReader.RunnableParser jenaStreamingParser
-
rdfIter
protected Iterator rdfIter
-
rdfInputStream
protected Stream rdfInputStream
-
lang
protected org.apache.jena.riot.Lang lang
-
collectionCount
protected int collectionCount
-
ignoreCollectionQuad
protected boolean ignoreCollectionQuad
-
hasOutputCol
protected boolean hasOutputCol
-
outputGraph
protected String outputGraph
-
outputOverrideGraph
protected String outputOverrideGraph
-
buffer
protected StringBuilder buffer
-
hasNext
protected boolean hasNext
-
idGen
protected IdGenerator idGen
-
random
protected Random random
-
randomValue
protected long randomValue
-
milliSecs
protected long milliSecs
-
origFn
protected String origFn
-
inputFn
protected String inputFn
-
splitStart
protected long splitStart
-
start
protected long start
-
pos
protected long pos
-
end
protected long end
-
compressed
protected boolean compressed
-
ingestedTriples
protected long ingestedTriples
-
version
protected String version
-
roleMap
protected LinkedMapWritable roleMap
-
defaultPerms
protected com.marklogic.xcc.ContentPermission[] defaultPerms
-
graphQry
protected StringBuilder graphQry
-
roleMapExists
protected boolean roleMapExists
-
graphSupported
protected boolean graphSupported
-
-
Constructor Detail
-
RDFReader
public RDFReader(String version, LinkedMapWritable roleMap)
-
-
Method Detail
-
close
public void close() throws IOException- Specified by:
closein interfaceAutoCloseable- Specified by:
closein interfaceCloseable- Specified by:
closein classImportRecordReader<VALUEIN>- Throws:
IOException
-
submitGraphQuery
protected void submitGraphQuery() throws IOException- Throws:
IOException
-
getProgress
public float getProgress() throws IOException, InterruptedException- Specified by:
getProgressin classImportRecordReader<VALUEIN>- Throws:
IOExceptionInterruptedException
-
initialize
public void initialize(org.apache.hadoop.mapreduce.InputSplit inSplit, org.apache.hadoop.mapreduce.TaskAttemptContext context) throws IOException, InterruptedException- Specified by:
initializein classImportRecordReader<VALUEIN>- Throws:
IOExceptionInterruptedException
-
initStream
protected void initStream(org.apache.hadoop.mapreduce.InputSplit inSplit) throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
initParser
protected void initParser(String fsname, long size) throws IOException
- Throws:
IOException
-
parse
protected void parse(String fsname, org.apache.hadoop.fs.FSDataInputStream in) throws IOException
- Throws:
IOException
-
loadModel
protected void loadModel(String fsname, InputStream in) throws IOException
- Throws:
IOException
-
write
protected void write(String str)
-
resource
protected String resource(org.apache.jena.graph.Node rsrc)
-
subject
protected String subject(org.apache.jena.graph.Node subj)
-
subject
protected String subject(org.apache.jena.rdf.model.Resource subj)
-
predicate
protected String predicate(org.apache.jena.graph.Node subj)
-
predicate
protected String predicate(org.apache.jena.rdf.model.Resource subj)
-
object
protected String object(org.apache.jena.graph.Node node)
-
setKey
protected void setKey()
-
nextKeyValue
public boolean nextKeyValue() throws IOException, InterruptedException- Specified by:
nextKeyValuein classImportRecordReader<VALUEIN>- Throws:
IOExceptionInterruptedException
-
initExistingMapPerms
public void initExistingMapPerms() throws IOException- Throws:
IOException
-
insertGraphDoc
public com.marklogic.xcc.ContentPermission[] insertGraphDoc(String graph) throws IOException
- Throws:
IOException
-
nextInMemoryKeyValue
public boolean nextInMemoryKeyValue() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextInMemoryTripleKeyValue
public boolean nextInMemoryTripleKeyValue() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextInMemoryQuadKeyValue
public boolean nextInMemoryQuadKeyValue() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextInMemoryQuadKeyValueWithCollections
public boolean nextInMemoryQuadKeyValueWithCollections() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextInMemoryQuadKeyValueIgnoreCollections
public boolean nextInMemoryQuadKeyValueIgnoreCollections() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextStreamingKeyValue
public boolean nextStreamingKeyValue() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextStreamingTripleKeyValue
protected boolean nextStreamingTripleKeyValue() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextStramingQuadKeyValue
public boolean nextStramingQuadKeyValue() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextStreamingQuadKeyValueIgnoreCollections
protected boolean nextStreamingQuadKeyValueIgnoreCollections() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
nextStreamingQuadKeyValueWithCollections
public boolean nextStreamingQuadKeyValueWithCollections() throws IOException, InterruptedException- Throws:
IOExceptionInterruptedException
-
writeValue
public void writeValue() throws IOException- Throws:
IOException
-
writeValue
public void writeValue(String collection) throws IOException
- Throws:
IOException
-
largestCollection
protected String largestCollection()
-
notifyUser
protected void notifyUser()
-
-