NationalSecurityAgency/datawave
1
0 Fork 0
mirror of https://github.com/NationalSecurityAgency/datawave.git synced 2026-11-11 03:31:32 -10:00
Code Issues Packages Projects Releases Wiki Activity
Files
task/replace-sortedmapiterator
datawave/properties/compose.properties
T
MoriartyandClaude Opus 5 0ac549a3a2 Pre-split the shard table in docker compose (#3885) 
The whole sharded schema sat in one tablet, so no number of tablet
servers, scan servers or compactors made any difference to a query.
Splits now come from a checked-in file, the deployment moves to three
shards per day, and verifySplits.sh guards that in CI.

Also fixes the num shards cache, which scanned a metadata table name no
deployment that renames its tables actually uses. The missing table was
swallowed, so a day expanded into no shards and the query silently
returned nothing.

---------

Co-authored-by: Claude Opus 5 <[email protected]>
2026-08-28 11:24:22 +00:00

343 lines
13 KiB
Properties
Raw Permalink Blame History

CONFIGURATION=test
RCPT_TO=hadoop@localhost
DW_CLOUD_BIN=/opt/datawave/contrib/datawave-quickstart/bin
DW_CLOUD_PLUGINS=${DW_CLOUD_BIN}/services
# ingest properties
DATAWAVE_INGEST_HOME=/opt/datawave-ingest/current
WAREHOUSE_ACCUMULO_HOME=/opt/accumulo
WAREHOUSE_HDFS_NAME_NODE=hdfs://hdfs-nn:9000
WAREHOUSE_JOBTRACKER_NODE=yarn-rm:8032
WAREHOUSE_ZOOKEEPERS=zookeeper:2181
WAREHOUSE_INSTANCE_NAME=my-instance-01
#Sets variable sets the zookeeper location for the warehouse side
zookeeper.hosts=zookeeper:2181
INGEST_ACCUMULO_HOME=/opt/accumulo
INGEST_HDFS_NAME_NODE=hdfs://hdfs-nn:9000
INGEST_JOBTRACKER_NODE=yarn-rm:8032
INGEST_ZOOKEEPERS=zookeeper:2181
INGEST_INSTANCE_NAME=my-instance-01
JOB_CACHE_REPLICATION=1
STAGING_HOSTS=localhost
DUMPER_HOSTS=localhost
INGEST_HOST=localhost
ROLLUP_HOST=localhost
#extra mapreduce options (e.g. mapreduce.task.io.sort.mb and the like)
MAPRED_INGEST_OPTS=-useInlineCombiner -ingestMetricsDisabled
#extra HADOOP_OPTS (java options)
HADOOP_INGEST_OPTS=
#extra CHILD_OPTS (java options)
CHILD_INGEST_OPTS=
BULK_CHILD_MAP_MAX_MEMORY_MB=2048
LIVE_CHILD_MAP_MAX_MEMORY_MB=1024
BULK_CHILD_REDUCE_MAX_MEMORY_MB=2048
LIVE_CHILD_REDUCE_MAX_MEMORY_MB=1024
BULK_INGEST_DATA_TYPES=shardStats,annotation
LIVE_INGEST_DATA_TYPES=wikipedia,mycsv,myjson,annotation
# Clear out these values if you do not want standard shard ingest.
DEFAULT_SHARD_HANDLER_CLASSES=datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler
ALL_HANDLER_CLASSES=datawave.ingest.mapreduce.handler.edge.ProtobufEdgeDataTypeHandler,datawave.ingest.mapreduce.handler.dateindex.DateIndexDataTypeHandler
BULK_INGEST_REDUCERS=10
LIVE_INGEST_REDUCERS=10
# Note the max blocks per job must be less than or equal to the number of mappers
INGEST_BULK_JOBS=1
INGEST_BULK_MAPPERS=4
INGEST_MAX_BULK_BLOCKS_PER_JOB=4
INGEST_LIVE_JOBS=1
INGEST_LIVE_MAPPERS=4
INGEST_MAX_LIVE_BLOCKS_PER_JOB=4
INDEX_STATS_MAX_MAPPERS=7
NUM_MAP_LOADERS=1
USERNAME=root
PASSWORD=secret
ZOOKEEPER_HOME=/opt/zookeeper
HADOOP_HOME=/usr/local/hadoop
MAPRED_HOME=/usr/local/hadoop-mapreduce
WAREHOUSE_HADOOP_CONF=/stack
INGEST_HADOOP_CONF=/stack
HDFS_BASE_DIR=/datawave/ingest
MONITOR_SERVER_HOST=accumulo-monitor
LOG_DIR=/srv/logs/ingest
PDSH_LOG_DIR=${LOG_DIR}/pdsh_logs
FLAG_DIR=/srv/data/datawave/flags
FLAG_MAKER_CONFIG=/opt/datawave-ingest/current/config/flag-maker-live.xml
BIN_DIR_FOR_FLAGS=/opt/datawave-ingest/current/bin
PYTHON=/usr/bin/python
# Setting discard interval to 0 in order to disable auto-ageoff @ ingest time
EVENT_DISCARD_INTERVAL=0
# Setting discard interval to 0 in order to disable auto-ageoff @ ingest time
EVENT_DISCARD_FUTURE_INTERVAL=0
DATAWAVE_CACHE_PORT=20444
EDGE_DEFINITION_FILE=config/edge-definitions.xml
ERROR_TABLE=errors
ANALYTIC_MTX=analytic_metrics
LOADER_MTX=loader_metrics
INGEST_MTX=ingest_metrics
BULK_INGEST_METRIC_THRESHOLD=1500000
LIVE_INGEST_METRIC_THRESHOLD=1500000
KEYSTORE=/opt/jboss/wildfly/standalone/configuration/certificates/testServer.p12
KEYSTORE_TYPE=PKCS12
KEYSTORE_PASSWORD=ChangeIt
TRUSTSTORE=/opt/jboss/wildfly/standalone/configuration/certificates/ca.jks
FLAG_METRICS_DIR=/srv/data/datawave/flagMetrics
TRUSTSTORE_PASSWORD=ChangeIt
TRUSTSTORE_TYPE=JKS
cluster.name=quickstart
accumulo.instance.name=my-instance-01
accumulo.user.name=root
accumulo.user.password=secret
cached.results.hdfs.uri=hdfs://hdfs-nn:9000
cached.results.export.dir=/CachedResults
lock.file.dir=/var/run/datawave
JAVA_HOME=/usr/lib/jvm/java
# query properties
server.keystore.password=secret
mysql.user.password=datawave
jboss.jmx.password=secret
hornetq.cluster.password=secret
hornetq.system.password=secret
server.truststore.password=Changeit
#Sets up the Atom Service
atom.wildfly.hostname=localhost
atom.wildfly.port.number=8443
atom.connection.pool.name=WAREHOUSE
# other properties
rpm.file.owner=rpmowner
rpm.file.group=rpmowner
rpm.file.accumulo.owner=accumulo-owner
rpm.file.accumulo.group=accumulo-owner
# Enable full table scans for the base event query?
#beq.fullTableScanEnabled=true
event.query.data.decorators= \
<entry key="CSV"> \
\n <bean class="datawave.query.transformer.EventQueryDataDecorator"> \
\n <property name="fieldName" value="CSV"/> \
\n <property name="patternMap"> \
\n <map key-type="java.lang.String" value-type="java.lang.String"> \
\n <entry key="EVENT_ID" value="https://localhost:8443/DataWave/Query/lookupUUID/EVENT_ID?uuid=@field_value@&amp;parameters=data.decorators:CSV"/> \
\n <entry key="UUID" value="https://localhost:8443/DataWave/Query/lookupUUID/UUID?uuid=@field_value@&amp;parameters=data.decorators:CSV"/> \
\n <entry key="PARENT_UUID" value="https://localhost:8443/DataWave/Query/lookupUUID/PARENT_UUID?uuid=@field_value@&amp;parameters=data.decorators:CSV"/> \
\n </map> \
\n </property> \
\n <property name="responseObjectFactory" ref="responseObjectFactory" /> \
\n </bean> \
\n </entry> \
\n <entry key="WIKIPEDIA"> \
\n <bean class="datawave.query.transformer.EventQueryDataDecorator"> \
\n <property name="fieldName" value="WIKIPEDIA"/> \
\n <property name="patternMap"> \
\n <map key-type="java.lang.String" value-type="java.lang.String"> \
\n <entry key="PAGE_ID" value="https://localhost:8443/DataWave/Query/lookupUUID/PAGE_ID?uuid=@field_value@&amp;parameters=data.decorators:WIKIPEDIA"/> \
\n <entry key="PAGE_TITLE" value="https://localhost:8443/DataWave/Query/lookupUUID/PAGE_TITLE?uuid=@field_value@&amp;parameters=data.decorators:WIKIPEDIA"/> \
\n </map> \
\n </property> \
\n <property name="responseObjectFactory" ref="responseObjectFactory" /> \
\n </bean> \
\n </entry>
lookup.uuid.uuidTypes= \
<bean class="datawave.query.data.UUIDType"> \
\n <property name="fieldName" value="ID" /> \
\n <property name="queryLogics" ref="DefaultUUIDQueryLogics" /> \
\n </bean> \
\n <bean class="datawave.query.data.UUIDType"> \
\n <property name="fieldName" value="EMBEDDED_CAST_PERSON_ID" /> \
\n <property name="queryLogics">\
\n <util:map>\
\n <entry key="default" value="LuceneUUIDEventQuery"/> \
\n </util:map> \
\n </property> \
\n </bean> \
\n <bean class="datawave.query.data.UUIDType"> \
\n <property name="fieldName" value="EVENT_ID" /> \
\n <property name="queryLogics" ref="DefaultUUIDQueryLogics" /> \
\n <property name="allowWildcardAfter" value="28" /> \
\n </bean> \
\n <bean class="datawave.query.data.UUIDType"> \
\n <property name="fieldName" value="UUID" /> \
\n <property name="queryLogics" ref="DefaultUUIDQueryLogics" /> \
\n </bean> \
\n <bean class="datawave.query.data.UUIDType"> \
\n <property name="fieldName" value="PARENT_UUID" /> \
\n <property name="queryLogics" ref="DefaultUUIDQueryLogics" /> \
\n </bean> \
\n <bean class="datawave.query.data.UUIDType"> \
\n <property name="fieldName" value="PAGE_ID" /> \
\n <property name="queryLogics" ref="DefaultUUIDQueryLogics" /> \
\n </bean> \
\n <bean class="datawave.query.data.UUIDType"> \
\n <property name="fieldName" value="PAGE_TITLE" /> \
\n <property name="queryLogics" ref="DefaultUUIDQueryLogics" /> \
\n </bean>
query.metrics.marking=(PUBLIC)
query.metrics.visibility=PUBLIC
metrics.warehouse.namenode=localhost
metrics.warehouse.hadoop.path=/local/hadoop
metrics.reporter.class=datawave.metrics.NoOpMetricsReporterFactory
metadatahelper.default.auths=PUBLIC
security.npe.ou.entries=EXAMPLE_SERVER_OU1,EXAMPLE_SERVER_OU2
security.subject.dn.pattern=(?:^|,)\\s*OU\\s*=\\s*My Department\\s*(?:,|$)
datawave.docs.menu.extras=<li><a href="http://localhost:9995">Accumulo</a></li>
type.metadata.hdfs.uri=hdfs://hdfs-nn:9000
mapReduce.hdfs.uri=hdfs://hdfs-nn:9000
bulkResults.hdfs.uri=hdfs://hdfs-nn:9000
jboss.log.hdfs.uri=hdfs://hdfs-nn:9000
jboss.managed.executor.service.default.max.threads=48
mapReduce.job.tracker=yarn-rm:8032
bulkResults.job.tracker=yarn-rm:8032
ingest.data.types=wikipedia,mycsv,myjson,shardStats
PASSWORD_INGEST_ENV=/opt/datawave-ingest/ingest-passwd.sh
hdfs.site.config.urls=file:///stack/core-site.xml,file:///stack/hdfs-site.xml
ivarator.cache.dir.config= \
<bean class="datawave.query.iterator.ivarator.IvaratorCacheDirConfig"> \
\n <constructor-arg value="hdfs://hdfs-nn:9000/IvaratorCache" /> \
\n </bean>
ivarator.fst.hdfs.base.uris=hdfs://hdfs-nn:9000/IvaratorCache
table.shard.numShardsPerDay=3
############################
#
# Security Settings
#
############################
# Whether or not to use the remote authorization service
security.use.remoteauthservice=true
# Whether or not to use the test authorization service that loads canned users
security.use.testauthservice=false
# Configuration for the remote DatawaveUser service
#
# Find the host and port of the service using a SRV DNS lookup
security.remoteuserservice.srv.lookup.enabled=false
# The DNS servers to use for the SRV lookup
security.remoteuserservice.srv.lookup.servers=127.0.0.1
# The port on which the DNS server that serves SRV records is listening
security.remoteuserservice.srv.lookup.port=8600
# The scheme to use when connecting to the remote user service
security.remoteuserservice.scheme=https
# The host to connect to (or do a SRV lookup on) for the remote user service
security.remoteuserservice.host=authorization
# The port to connect to (unless a SRV lookup was performed) for the remote user service
security.remoteuserservice.port=8443
############################
#
# Audit Settings
#
############################
# Whether or not to use the remote audit service
auditing.use.remoteauditservice=true
# Configuration for the remote audit service
#
# Find the host and port of the service using a SRV DNS lookup
auditing.remoteauditservice.srv.lookup.enabled=false
# The DNS servers to use for the SRV lookup
auditing.remoteauditservice.srv.lookup.servers=127.0.0.1
# The port on which the DNS server that serves SRV records is listening
auditing.remoteauditservice.srv.lookup.port=8600
# The scheme to use when connecting to the remote audit service
auditing.remoteauditservice.scheme=https
# The host to connect to (or do a SRV lookup on) for the remote audit service
auditing.remoteauditservice.host=audit
# The port to connect to (unless a SRV lookup was performed) for the remote audit service
auditing.remoteauditservice.port=8443
############################
#
# Dictionary Settings
#
############################
# Configuration for the remote dictionary service
#
# The scheme to use when connecting to the remote dictionary service
dictionary.remoteservice.scheme=https
# The host to connect to (or do a SRV lookup on) for the remote dictionary service
dictionary.remoteservice.host=localhost
# The port to connect to (unless a SRV lookup was performed) for the remote dictionary service
dictionary.remoteservice.port=8643
############################
#
# Configuration for the remote Accumulo service
#
############################
# Find the host and port of the service using a SRV DNS lookup
accumulo.remoteservice.srv.lookup.enabled=false
# The DNS servers to use for the SRV lookup
accumulo.remoteservice.srv.lookup.servers=127.0.0.1
# The port on which the DNS server that serves SRV records is listening
accumulo.remoteservice.srv.lookup.port=8600
# The scheme to use when connecting to the remote user service
accumulo.remoteservice.scheme=https
# The host to connect to (or do a SRV lookup on) for the remote user service
accumulo.remoteservice.host=accumulo
# The port to connect to (unless a SRV lookup was performed) for the remote user service
accumulo.remoteservice.port=8443
############################
#
# Configuration for the remote Query Metric service
#
############################
# Whether or not to use the remote query metric service
querymetric.remoteservice.enabled=true
# Find the host and port of the service using a SRV DNS lookup
querymetric.remoteservice.srv.lookup.enabled=false
# The DNS servers to use for the SRV lookup
querymetric.remoteservice.srv.lookup.servers=127.0.0.1
# The port on which the DNS server that serves SRV records is listening
querymetric.remoteservice.srv.lookup.port=8600
# The scheme to use when connecting to the remote query metric service
querymetric.remoteservice.scheme=https
# The host to connect to (or do a SRV lookup on) for the remote query metric service
querymetric.remoteservice.host=metrics
# The port to connect to (unless a SRV lookup was performed) for the remote query metric service
querymetric.remoteservice.port=8443
Reference in new issue View Git Blame Copy Permalink