Package com.marklogic.contentpump
Class DatabaseContentWriter<VALUE>
- java.lang.Object
-
- org.apache.hadoop.mapreduce.RecordWriter<KEYOUT,VALUEOUT>
-
- com.marklogic.mapreduce.MarkLogicRecordWriter<DocumentURI,VALUEOUT>
-
- com.marklogic.mapreduce.ContentWriter<VALUE>
-
- com.marklogic.contentpump.DatabaseContentWriter<VALUE>
-
- All Implemented Interfaces:
ConfigConstants,MarkLogicConstants
public class DatabaseContentWriter<VALUE> extends ContentWriter<VALUE> implements ConfigConstants
MarkLogicRecordWriter that can 1) insert content from Archive to MarkLogic Server 2) copy content from source MarkLogic Server to destination MarkLogic Server
-
-
Field Summary
Fields Modifier and Type Field Description protected booleanisCopyPermsprotected booleanisCopyPropsstatic org.apache.commons.logging.LogLOGprotected int[]propertyCountsprotected com.marklogic.xcc.types.XdmValue[][]propertyUrisprotected com.marklogic.xcc.types.XdmValue[][]propertyXmlStringsstatic StringXQUERY_VERSION_1_0_ML-
Fields inherited from class com.marklogic.mapreduce.ContentWriter
am, batchId, batchRetry, batchSize, batchSleepTime, blacklist, commitRetry, commitRetryLimit, commitSleepTime, commitUris, countBased, counts, curReplica, effectiveVersion, failed, fastLoad, forestContents, forestIds, hostId, hostSourceMap, ID_PREFIX, is, isCopyColls, isCopyMeta, isCopyQuality, isTxnCompatible, MAX_RETRIES, MAX_SLEEP_TIME, MIN_RETRIES, MIN_SLEEP_TIME, needCommit, options, outputDir, pendingUris, permsMap, roleMap, sessions, sfId, stmtCounts, succeeded
-
Fields inherited from class com.marklogic.mapreduce.MarkLogicRecordWriter
conf, hostName, txnSize
-
Fields inherited from interface com.marklogic.contentpump.ConfigConstants
AGGREGATE_RECORD_ELEMENT, AGGREGATE_RECORD_NAMESPACE, AGGREGATE_URI_ID, API_KEY, ARCHIVE_METADATA_OPTIONAL, AUDIT_MLCPFINISH_CODE, AUDIT_MLCPFINISH_EVENT, AUDIT_MLCPSTART_CODE, AUDIT_MLCPSTART_EVENT, BASE_PATH, BATCH_MIN_VERSION, BATCH_SIZE, COLLECTION_FILTER, CONF_AGGREGATE_RECORD_ELEMENT, CONF_AGGREGATE_RECORD_NAMESPACE, CONF_AUDIT_MLCPFINISH_ENABLED, CONF_AUDIT_MLCPFINISH_MESSAGE, CONF_AUDIT_MLCPSTART_MESSAGE, CONF_COPY_PERMISSIONS, CONF_COPY_PROPERTIES, CONF_DELIMITED_DATA_TYPE, CONF_DELIMITED_ROOT_NAME, CONF_DELIMITER, CONF_INPUT_ARCHIVE_METADATA_OPTIONAL, CONF_INPUT_COMPRESSION_CODEC, CONF_INPUT_DIRECTORY, CONF_INPUT_FILE_PATTERN, CONF_INPUT_GENERATE_URI, CONF_INPUT_MODULES_DATABASE, CONF_INPUT_MODULES_ROOT, CONF_INPUT_PATH_FILTER_CLASS, CONF_INPUT_SEQUENCEFILE_KEY_CLASS, CONF_INPUT_SEQUENCEFILE_VALUE_CLASS, CONF_INPUT_SEQUENCEFILE_VALUE_TYPE, CONF_INPUT_URI_ID, CONF_MAPREDUCE_JOB_MAP_CLASS, CONF_MAPREDUCE_JOB_WORKING_DIR, CONF_MAPREDUCE_JOBTRACKER_ADDRESS, CONF_MAX_SPLIT_SIZE1, CONF_MAX_SPLIT_SIZE2, CONF_MIMETYPES, CONF_MIN_SPLIT_SIZE1, CONF_MIN_SPLIT_SIZE2, CONF_MIN_THREADS, CONF_ML_VERSION, CONF_MULTITHREADEDMAPPER_CLASS, CONF_OUTPUT_FILENAME_AS_COLLECTION, CONF_OUTPUT_FILEPATH, CONF_OUTPUT_TYPE, CONF_ROLE_MAP, CONF_SPLIT_INPUT, CONF_THREADS_PER_SPLIT, CONF_TRANSFORM_FUNCTION, CONF_TRANSFORM_MODULE, CONF_TRANSFORM_NAMESPACE, CONF_TRANSFORM_PARAM, CONTENT_ENCODING, CONTENTPUMP_BUNDLE_ARTIFACT, CONTENTPUMP_HOME_PROPERTY_NAME, CONTENTPUMP_JAR_PREFIX, COPY_COLLECTIONS, COPY_METADATA, COPY_PERMISSIONS, COPY_PROPERTIES, COPY_QUALITY, DATA_TYPE, DATABASE, DEFAULT_ARCHIVE_METADATA_OPTIONAL, DEFAULT_COPY_COLLECTIONS, DEFAULT_COPY_METADATA, DEFAULT_COPY_PERMISSIONS, DEFAULT_COPY_PROPERTIES, DEFAULT_COPY_QUALITY, DEFAULT_DELIMITER, DEFAULT_ENCODING, DEFAULT_ML_CLOUD_PORT, DEFAULT_OUTPUT_TYPE, DEFAULT_PORT, DEFAULT_SEQUENCEFILE_VALUE_TYPE, DEFAULT_THREAD_COUNT, DELIMITED_ROOT_NAME, DELIMITED_URI_ID, DELIMITER, DIRECTORY_FILTER, DOCUMENT_SELECTOR, DOCUMENT_TYPE, FAST_LOAD, GENERATE_URI, GRANT_TYPE, HADOOP_CONF_DIR, HADOOP_CONFDIR_ENV_NAME, HOST, INPUT_API_KEY, INPUT_BASE_PATH, INPUT_COMPRESSED, INPUT_COMPRESSION_CODEC, INPUT_DATABASE, INPUT_FILE_PATH, INPUT_FILE_PATTERN, INPUT_FILE_TYPE, INPUT_FILE_TYPE_DEFAULT, INPUT_GRANT_TYPE, INPUT_HOST, INPUT_KEYSTORE_PASSWD, INPUT_KEYSTORE_PATH, INPUT_PASSWORD, INPUT_PORT, INPUT_SEQUENCEFILE_KEY_CLASS, INPUT_SEQUENCEFILE_VALUE_CLASS, INPUT_SEQUENCEFILE_VALUE_TYPE, INPUT_SSL, INPUT_SSL_PROTOCOL, INPUT_TOKEN_DURATION, INPUT_TOKEN_ENDPOINT, INPUT_TRUSTSTORE_PASSWD, INPUT_TRUSTSTORE_PATH, INPUT_USERNAME, KEYSTORE_PASSWD, KEYSTORE_PATH, MAX_BATCH_SIZE, MAX_SPLIT_SIZE, MAX_THREAD_PERCENTAGE, MAX_THREADS, MAX_TXN_SIZE, MIN_SPLIT_SIZE, MODE, MODULES, MODULES_ROOT, NAMESPACE, OPTIONS_FILE, OUTPUT_API_KEY, OUTPUT_BASE_PATH, OUTPUT_CLEANDIR, OUTPUT_COLLECTIONS, OUTPUT_COMPRESS, OUTPUT_DATABASE, OUTPUT_DIRECTORY, OUTPUT_FILE_PATH, OUTPUT_FILENAME_AS_COLLECTION, OUTPUT_GRANT_TYPE, OUTPUT_GRAPH, OUTPUT_HOST, OUTPUT_IDNAME, OUTPUT_INDENTED, OUTPUT_KEYSTORE_PASSWD, OUTPUT_KEYSTORE_PATH, OUTPUT_LANGUAGE, OUTPUT_OVERRIDE_GRAPH, OUTPUT_PARTITION, OUTPUT_PASSWORD, OUTPUT_PERMISSIONS, OUTPUT_PORT, OUTPUT_QUALITY, OUTPUT_SSL, OUTPUT_SSL_PROTOCOL, OUTPUT_TOKEN_DURATION, OUTPUT_TOKEN_ENDPOINT, OUTPUT_TRUSTSTORE_PASSWD, OUTPUT_TRUSTSTORE_PATH, OUTPUT_TYPE, OUTPUT_URI_PREFIX, OUTPUT_URI_REPLACE, OUTPUT_URI_SUFFIX, OUTPUT_USERNAME, PASSWORD, PATH_NAMESPACE, POLLING_INIT_DELAY, POLLING_PERIOD, POLLING_TIME_UNIT, PORT, QUERY_FILTER, RDF_STREAMING_MEMORY_THRESHOLD, RDF_TRIPLES_PER_DOCUMENT, REDACTION, RESTRICT_HOSTS, RESTRICT_INPUT_HOSTS, RESTRICT_OUTPUT_HOSTS, SNAPSHOT, SPLIT_INPUT, SSL, SSL_PROTOCOL, STREAMING, TEMPORAL_COLLECTION, THREAD_COUNT, THREADS_PER_SPLIT, TOKEN_DURATION, TOKEN_ENDPOINT, TRANSACTION_SIZE, TRANSFORM_FUNCTION, TRANSFORM_MODULE, TRANSFORM_NAMESPACE, TRANSFORM_PARAM, TRUSTSTORE_PASSWD, TRUSTSTORE_PATH, TYPE_FILTER, URI_ID, USERNAME, XML_REPAIR_LEVEL
-
Fields inherited from interface com.marklogic.mapreduce.MarkLogicConstants
ADVANCED_MODE, ASSIGNMENT_POLICY, BASIC_MODE, BATCH_SIZE, BIND_SPLIT_RANGE, COLLECTION_FILTER, CONTENT_TYPE, COPY_COLLECTIONS, COPY_METADATA, COPY_QUALITY, DEFAULT_BATCH_SIZE, DEFAULT_CONTENT_TYPE, DEFAULT_LOCAL_MAX_SPLIT_SIZE, DEFAULT_MAX_SPLIT_SIZE, DEFAULT_OUTPUT_CONTENT_ENCODING, DEFAULT_OUTPUT_XML_REPAIR_LEVEL, DEFAULT_PROPERTY_OPERATION_TYPE, DEFAULT_TXN_SIZE, DIRECTORY_FILTER, DOCUMENT_SELECTOR, EXECUTION_MODE, EXTRACT_URI, INDENTED, INPUT_API_KEY, INPUT_BASE_PATH, INPUT_DATABASE_NAME, INPUT_GRANT_TYPE, INPUT_HOST, INPUT_KEY_CLASS, INPUT_KEYSTORE_PASSWD, INPUT_KEYSTORE_PATH, INPUT_LEXICON_FUNCTION_CLASS, INPUT_MODE, INPUT_PASSWORD, INPUT_PORT, INPUT_QUERY, INPUT_QUERY_LANGUAGE, INPUT_QUERY_TIMESTAMP, INPUT_RESTRICT_HOSTS, INPUT_SSL_OPTIONS_CLASS, INPUT_SSL_PROTOCOL, INPUT_TOKEN_DURATION, INPUT_TOKEN_ENDPOINT, INPUT_TRUSTSTORE_PASSWD, INPUT_TRUSTSTORE_PATH, INPUT_USE_SSL, INPUT_USERNAME, INPUT_VALUE_CLASS, MAX_SPLIT_SIZE, MIN_NODEUPDATE_VERSION, MODE_DISTRIBUTED, MODE_LOCAL, MR_NAMESPACE, NODE_OPERATION_TYPE, OUTPUT_API_KEY, OUTPUT_BASE_PATH, OUTPUT_CLEAN_DIR, OUTPUT_COLLECTION, OUTPUT_CONTENT_ENCODING, OUTPUT_CONTENT_LANGUAGE, OUTPUT_CONTENT_NAMESPACE, OUTPUT_DATABASE_NAME, OUTPUT_DIRECTORY, OUTPUT_FAST_LOAD, OUTPUT_FOREST_HOST, OUTPUT_GRANT_TYPE, OUTPUT_GRAPH, OUTPUT_HOST, OUTPUT_KEY_TYPE, OUTPUT_KEY_VARNAME, OUTPUT_KEYSTORE_PASSWD, OUTPUT_KEYSTORE_PATH, OUTPUT_NAMESPACE, OUTPUT_OVERRIDE_GRAPH, OUTPUT_PARTITION, OUTPUT_PASSWORD, OUTPUT_PERMISSION, OUTPUT_PORT, OUTPUT_PROPERTY_ALWAYS_CREATE, OUTPUT_QUALITY, OUTPUT_QUERY, OUTPUT_QUERY_LANGUAGE, OUTPUT_RESTRICT_HOSTS, OUTPUT_SSL_OPTIONS_CLASS, OUTPUT_SSL_PROTOCOL, OUTPUT_STREAMING, OUTPUT_TOKEN_DURATION, OUTPUT_TOKEN_ENDPOINT, OUTPUT_TRUSTSTORE_PASSWD, OUTPUT_TRUSTSTORE_PATH, OUTPUT_URI_PREFIX, OUTPUT_URI_REPLACE, OUTPUT_URI_SUFFIX, OUTPUT_USE_SSL, OUTPUT_USERNAME, OUTPUT_VALUE_TYPE, OUTPUT_VALUE_VARNAME, OUTPUT_XML_REPAIR_LEVEL, PATH_NAMESPACE, PROPERTY_OPERATION_TYPE, QUERY_FILTER, RECORD_TO_FRAGMENT_RATIO, REDACTION_RULE_COLLECTION, SPLIT_END_VARNAME, SPLIT_QUERY, SPLIT_START_VARNAME, SUBDOCUMENT_EXPRESSION, TEMPORAL_COLLECTION, TXN_SIZE, TYPE_FILTER
-
-
Constructor Summary
Constructors Constructor Description DatabaseContentWriter(org.apache.hadoop.conf.Configuration conf, Map<String,com.marklogic.xcc.ContentSource> hostSourceMap, boolean fastLoad)DatabaseContentWriter(org.apache.hadoop.conf.Configuration conf, Map<String,com.marklogic.xcc.ContentSource> hostSourceMap, boolean fastLoad, AssignmentManager am)
-
Method Summary
All Methods Static Methods Instance Methods Concrete Methods Modifier and Type Method Description voidclose(org.apache.hadoop.mapreduce.TaskAttemptContext context)protected static com.marklogic.xcc.ContentCreateOptionsnewContentCreateOptions(DocumentMetadata meta, com.marklogic.xcc.ContentCreateOptions options, boolean isCopyColls, boolean isCopyQuality, boolean isCopyMeta, boolean isCopyPerms, long effectiveVersion)fetch the options information from conf and metadata, set to the field "options"protected booleansetBatchProperties(int i, com.marklogic.xcc.Session s)protected static booleansetDocumentProperties(String uri, String xmlString, String permString, String collString, String quality, Map<String,String> meta, com.marklogic.xcc.Session s)voidwrite(DocumentURI key, VALUE value)-
Methods inherited from class com.marklogic.mapreduce.ContentWriter
closeSessions, commit, createContent, getFormattedBatchId, getSession, getSession, getTransactionSize, handleCommitExceptions, insertBatch, needCommit, needCommitRetry, rollback, rollbackCount, sleep
-
Methods inherited from class com.marklogic.mapreduce.MarkLogicRecordWriter
commitIfNecessary, getSession
-
-
-
-
Field Detail
-
LOG
public static final org.apache.commons.logging.Log LOG
-
propertyUris
protected com.marklogic.xcc.types.XdmValue[][] propertyUris
-
propertyXmlStrings
protected com.marklogic.xcc.types.XdmValue[][] propertyXmlStrings
-
propertyCounts
protected int[] propertyCounts
-
isCopyProps
protected boolean isCopyProps
-
isCopyPerms
protected boolean isCopyPerms
-
XQUERY_VERSION_1_0_ML
public static final String XQUERY_VERSION_1_0_ML
- See Also:
- Constant Field Values
-
-
Constructor Detail
-
DatabaseContentWriter
public DatabaseContentWriter(org.apache.hadoop.conf.Configuration conf, Map<String,com.marklogic.xcc.ContentSource> hostSourceMap, boolean fastLoad)
-
DatabaseContentWriter
public DatabaseContentWriter(org.apache.hadoop.conf.Configuration conf, Map<String,com.marklogic.xcc.ContentSource> hostSourceMap, boolean fastLoad, AssignmentManager am)
-
-
Method Detail
-
newContentCreateOptions
protected static com.marklogic.xcc.ContentCreateOptions newContentCreateOptions(DocumentMetadata meta, com.marklogic.xcc.ContentCreateOptions options, boolean isCopyColls, boolean isCopyQuality, boolean isCopyMeta, boolean isCopyPerms, long effectiveVersion)
fetch the options information from conf and metadata, set to the field "options"
-
write
public void write(DocumentURI key, VALUE value) throws IOException, InterruptedException
- Overrides:
writein classContentWriter<VALUE>- Throws:
IOExceptionInterruptedException
-
close
public void close(org.apache.hadoop.mapreduce.TaskAttemptContext context) throws IOException, InterruptedException- Overrides:
closein classContentWriter<VALUE>- Throws:
IOExceptionInterruptedException
-
setBatchProperties
protected boolean setBatchProperties(int i, com.marklogic.xcc.Session s)
-
setDocumentProperties
protected static boolean setDocumentProperties(String uri, String xmlString, String permString, String collString, String quality, Map<String,String> meta, com.marklogic.xcc.Session s)
- Parameters:
uri- uri of the document whose property is to be setxmlString- property in xml stringpermString-collString-quality-meta-s-- Returns:
-
-