EIC code displayed by LXR master/​harvester/​templates/​panda/​panda_harvester.cfg.rpmnew.template	Source navigation Diff markup Identifier search General search
	Version: master test Revert   Change

Warning, /harvester/templates/panda/panda_harvester.cfg.rpmnew.template is written in an unsupported language. File is not indexed.

 #########################
 #
 # Master parameters
 #
 
 [master]
 
 # user name of the daemon process
 uname = FIXME
 
 # group name of the daemon process
 gname = FIXME
 
 # logger name
 loggername = harvester
 
 # harvester id - unique id as registered also in panda server
 harvester_id = FIXME
 
 # port number of debugger
 debugger_port = 19550
 
 # capability to dynamically change plugins
 dynamic_plugin_change = False
 
 
 
 
 ##########################
 #
 # Database parameters
 #
 
 [db]
 
 # verbose
 verbose = False
 
 # use inspect for decorator of verbose messages
 useInspect = False
 
 # number of database connections in each process
 nConnections = 10
 
 # database engine : sqlite or mariadb
 engine = sqlite
 
 # database filename for sqlite. Better to use local disk if possible since sqlite doesn't like NAS
 database_filename = FIXME
 
 # use MySQLdb for mariadb access
 useMySQLdb = False
 
 # user name for MariaDB. N/A for sqlite
 user = harvester
 
 # password for MariaDB. N/A for sqlite
 password = FIXME
 
 # schema for MariaDB. N/A for sqlite
 schema = HARVESTER
 
 # host name for MariaDB. N/A for sqlite
 host = localhost
 
 # port number for MariaDB. N/A for sqlite
 port =  3306
 
 # max time in seconds to keep trying to reconnect DB before timeout
 reconnectTimeout = 300
 
 # synchronize max workerID when starting up
 syncMaxWorkerID = False
 
 
 
 
 ##########################
 #
 # FIFO parameters
 #
 
 [fifo]
 
 # module and class names to provide the fifo queue
 fifoModule = pandaharvester.harvesterfifo.sqlite_fifo
 fifoClass = SqliteFifo
 
 # database filename for sqlite fifo plugin
 # must be different from main Harvester DB and other fifo DBs if using sqlite
 # different db filenames for fifos of different titles
 # a titile can be named after a harvester agent (e.g. monitor) or other special components
 # placeholder $(TITLE) should be used in filename; it will then be changed to the title name
 database_filename = /dev/shm/$(TITLE)_fifo.db
 
 
 
 
 ##########################
 #
 # Communicator parameters
 #
 
 [communicator]
 
 # module name of Communicator
 moduleName = pandaharvester.harvestercommunicator.panda_communicator
 
 # class name of Communicator
 className = PandaCommunicator
 
 # number of connections
 nConnections = 5
 
 
 
 
 ##########################
 #
 # Panda Connection parameters
 # (required only when PandaCommunicator is used to communicate with WMS)
 #
 
 [pandacon]
 
 # timeout
 timeout = 180
 
 # auth type x509 (default) or oidc
 auth_type = x509
 
 # CA file
 ca_cert = /etc/pki/tls/certs/CERN-bundle.pem
 
 # certificate for x509
 cert_file = FIXME
 
 # key for x509
 key_file = FIXME
 
 # token for oidc. bare string or filename (file:/path)
 auth_token = FIXME
 
 # origin for oidc
 auth_origin = FIXME
 
 # base URL via http
 pandaURL = http://pandaserver.cern.ch:25080/server/panda
 server_api_url = http://pandaserver.cern.ch:25080/api/v1
 
 # base URL via https
 pandaURLSSL = https://pandaserver.cern.ch:25443/server/panda
 server_api_url_ssl = https://pandaserver.cern.ch:25443/api/v1
 
 # base URL for write access to log cache server
 pandaCacheURL_W = https://aipanda011.cern.ch:25443/server/panda
 cache_api_url_ssl = https://pandacache.cern.ch:25443/api/v1
 
 # base URL for read access to log cache server
 pandaCacheURL_R = https://aipanda011.cern.ch:25443/cache
 
 # verbose
 verbose = False
 
 # use inspect for decorator of verbose messages
 useInspect = False
 
 # event size when getting events
 getEventsChunkSize = 5120
 
 # configuration file to support multiple auth types with various hosts: a json dump of
 # {"host:port": {"auth_type": "x509 or oidc", "cert_file": /path/to/cert, "key_file": /path/to/key,
 #                "ca_cert": /path/to/ca_cert, "auth_token": "token or file:/path/to/token"},
 #  ...}
 # the default values specified in cfg are used if the host:port or keys are not found in the json
 
 #multihost_auth_config = /path/to/multihost_auth_config.json
 
 
 ##########################
 #
 # Queue Config parameters
 #
 
 [qconf]
 
 # config file
 configFile = panda_queueconfig.json
 
 # enable config from cacher "queues_config_file"
 configFromCacher = False
 
 # queue list : one queue name following a whitespace per line. Or just ALL if all queues in the configFile are used
 queueList =
  FIXME_1
  FIXME_2
 
 # module and class names to resolve queue names to panda queue names
 resolverModule = pandaharvester.harvestermisc.info_utils
 resolverClass = PandaQueuesDict
 
 # configuration of the resolver in JSON
 #resolverConfig =
 #  {
 #    "refreshPeriod": 300
 #  }
 
 # enable auto-blacklisting of resolver which returns status='offline' to blacklist the queue
 autoBlacklist = False
 
 # restrict to a certain pilot version (optional)
 #pilotVersion = 2
 
 # update interval in sec (default: 600) - period to update qconf
 updateInterval = 600
 
 # check interval in sec (default: 5) - period for other agent threads to check last qconf update
 checkInterval = 5
 
 
 ##########################
 #
 # Command manager parameters
 #
 [commandmanager]
 
 # bulk size for panda server interactions
 commands_bulk_size = 20
 
 # sleep interval in sec
 sleepTime = 5
 
 
 
 
 ##########################
 #
 # Job Fetcher parameters
 #
 
 [jobfetcher]
 
 # number of threads
 nThreads = 3
 
 # number of queues to fetch jobs in one cycle
 nQueues = 5
 
 # max number of jobs in one cycle
 maxJobs = 500
 
 # lookup interval in sec
 lookupTime = 60
 
 # sleep interval in sec
 sleepTime = 60
 
 
 
 
 ##########################
 #
 # Propagator parameters
 #
 
 [propagator]
 
 # number of threads
 nThreads = 3
 
 # max number of jobs to update in one cycle
 maxJobs = 100
 
 # number of jobs in bulk update
 nJobsInBulk = 100
 
 # max number of workers to update in one cycle
 maxWorkers = 100
 
 # number of workers in bulk update
 nWorkersInBulk = 100
 
 # number of dialog message to send
 maxDialogs = 50
 
 # minimum level of dialog messages to send. INFO, WARNING, or ERROR
 minMessageLevel = WARNING
 
 # lock interval in sec
 lockInterval = 600
 
 # update interval in sec
 updateInterval = 1800
 
 # sleep interval in sec
 sleepTime = 60
 
 
 
 
 ##########################
 #
 # Preparator parameters
 #
 
 [preparator]
 
 # number of threads
 nThreads = 3
 
 # max number of jobs to check in one cycle
 maxJobsToCheck = 100
 
 # max number of jobs to trigger in one cycle
 maxJobsToTrigger = 100
 
 # max number of files per job to check in one cycle : 0 to be unlimited
 maxFilesPerJobToCheck = 0
 
 # max number of files per job to prepare in one cycle : 0 to be unlimited
 maxFilesPerJobToPrepare = 0
 
 # lock interval in sec
 lockInterval = 600
 
 # check interval in sec
 checkInterval = 180
 
 # trigger interval in sec
 triggerInterval = 180
 
 # sleep interval in sec
 sleepTime = 60
 
 
 
 
 ##########################
 #
 # Submitter parameters
 #
 
 [submitter]
 
 # number of threads
 nThreads = 3
 
 # max number of queues to try in one cycle
 nQueues = 3
 
 # interval for queue lookup
 lookupTime = 60
 
 # interval for queue lock
 queueLockInterval = 300
 
 # lock interval in sec
 lockInterval = 600
 
 # check interval in sec
 checkInterval = 60
 
 # minimum interval in sec between submissions to the same queue
 minSubmissionInterval = 0
 
 # sleep interval in sec
 sleepTime = 60
 
 # max number of workers per queue to try in one cycle
 maxNewWorkers = 1000
 
 # respect sleep time
 respectSleepTime = False
 
 # factor to adjust workers
 #activateWorkerFactor = auto
 
 ##########################
 #
 # Monitor parameters
 #
 
 [monitor]
 
 # number of threads
 nThreads = 3
 
 # max number of workers to try in one cycle
 maxWorkers = 500
 
 # lock interval in sec
 lockInterval = 600
 
 # check interval in sec
 checkInterval = 300
 
 # timeout in sec to give up checking if it keeps failing
 checkTimeout = 3600
 
 # sleep interval in sec
 sleepTime = 600
 
 # whether to use fifo
 fifoEnable = False
 
 # sleep interval in millisecond using fifo
 fifoSleepTimeMilli = 15000
 
 # check interval in fifo in sec
 fifoCheckInterval = 120
 
 # check duration of a fifo cycle in sec
 fifoCheckDuration = 60
 
 # interval of force enqueue in sec
 fifoForceEnqueueInterval = 1500
 
 # max number of workers to be populated into fifo
 #fifoMaxWorkersToPopulate = 100000
 
 # max number of workers in a chunk to enqueue
 fifoMaxWorkersPerChunk = 500
 
 # max interval in sec a post-processing worker can preempt in fifo
 fifoMaxPreemptInterval = 60
 
 # plugin cache parameters (used if monitor plugin supports)
 #pluginCacheEnable = True
 #pluginCacheRefreshInterval = 300
 
 # workers will be killed if stuck queuing (submitted) for longer than this
 workerQueueTimeLimit = 172800
 
 # enable event-based monitor check. Only works when fifoEnable is True
 eventBasedEnable = False
 
 # list of plugins for event-based check. Mandatory when eventBasedEnable is True
 #eventBasedPlugins =
 #  [
 #    {
 #      "module": "pandaharvester.harvestermonitor.htcondor_monitor",
 #      "name": "HTCondorMonitor",
 #      "condorHostConfig_list": [
 #          "/opt/harvester/etc/panda/condor_host_config.json"
 #        ]
 #    }
 #  ]
 
 # interval of event-based check to query with plugin, in sec
 #eventBasedCheckInterval = 300
 
 # time window of event-based check to check within, in sec
 #eventBasedTimeWindow = 450
 
 # max number of events of event-based check to handle in one cycle
 #eventBasedCheckMaxEvents = 500
 
 # lifetime of an event in event-based check, in sec
 #eventBasedEventLifetime = 1800
 
 # max number of expired events to remove in one cycle
 #eventBasedRemoveMaxEvents = 2000
 
 # timeout for post-processing in minutes. 0 to give up immediately
 postProcessTimeout = 0
 
 
 
 
 
 ##########################
 #
 # Credential Manager parameters
 #
 # Notes : This is an example to manage two credentials, one with production role and the other with pilot role.
 #         One credential data following a whitespace per line. Empty lines are not allowed, so that a dummy string
 #         like 'dummy' needs to be added if some parameters like voms are unnecessary.
 
 [credmanager]
 
 # module name
 moduleName =
  pandaharvester.harvestercredmanager.no_voms_cred_manager
  pandaharvester.harvestercredmanager.no_voms_cred_manager
 
 # class name
 className =
  NoVomsCredManager
  NoVomsCredManager
 
 # original certificate file to generate new short-lived certificate
 inCertFile =
  /path_to/FIXME_original_cert_for_proxy_with_production_role
  /path_to/FIXME_original_cert_for_proxy_with_pilot_role
 
 # the name of short-lived certificate
 outCertFile =
  /path_to/FIXME_proxy_production
  /path_to/FIXME_proxy_pilot
 
 # voms
 voms =
  atlas:/atlas/Role=production
  atlas:/atlas/Role=pilot
 
 # plugin configs in json
 # pluginConfigs =
 #   [
 #     {
 #       "module": "pandaharvester.harvestercredmanager.no_voms_cred_manager",
 #       "name": "NoVomsCredManager",
 #       "configs": {
 #         "production": {
 #          "inCertFile": "/data/atlpan/proxy/atlpilo1RFC.plain",
 #          "outCertFile": "/data/atlpan/proxy/x509up_u25606_prod",
 #          "voms": "atlas:/atlas/Role=production"
 #         },
 #         "pilot": {
 #           "inCertFile": "/data/atlpan/proxy/atlpilo1RFC.plain",
 #           "outCertFile": "/data/atlpan/proxy/x509up_u25606_pilot",
 #           "voms": "atlas:/atlas/Role=pilot"
 #         }
 #       }
 #     }
 #   ]
 
 
 # sleep interval in sec
 sleepTime = 1800
 
 
 
 
 
 ##########################
 #
 # Stager parameters
 #
 
 [stager]
 
 # number of threads
 nThreads = 3
 
 # max number of jobs to check in one cycle
 maxJobsToCheck = 100
 
 # max number of jobs to trigger in one cycle
 maxJobsToTrigger = 100
 
 # max number of jobs to zip in one cycle : OBSOLETE, should be set in [zipper]
 maxJobsToZip = 100
 
 # max number of files per job to check in one cycle : 0 to be unlimited
 maxFilesPerJobToCheck = 0
 
 # max number of files per job to trigger stage-out in one cycle : 0 to be unlimited
 maxFilesPerJobToTrigger = 0
 
 # max number of files per job to zip in one cycle : 0 to be unlimited : OBSOLETE, should be set in [zipper]
 maxFilesPerJobToZip = 0
 
 # use two staged zipping : OBSOLETE, should be set in [zipper]
 usePostZipping = False
 
 # lock interval in sec
 lockInterval = 600
 
 # check interval in sec
 checkInterval = 180
 
 # trigger interval in sec
 triggerInterval = 180
 
 # zip interval in sec : OBSOLETE, should be set in [zipper]
 zipInterval = 180
 
 # number of threads for zip making : OBSOLETE, should be set in [zipper]
 nThreadsForZip = 4
 
 # sleep interval in sec
 sleepTime = 60
 
 
 
 
 
 ##########################
 #
 # Zipper parameters
 #
 
 [zipper]
 
 # max number of jobs to zip in one cycle
 maxJobsToZip = 100
 
 # max number of files per job to zip in one cycle : 0 to be unlimited
 maxFilesPerJobToZip = 0
 
 # use two staged zipping
 usePostZipping = False
 
 # lock interval in sec
 lockInterval = 600
 
 # zip interval in sec
 zipInterval = 180
 
 # number of threads for zip making
 nThreadsForZip = 4
 
 
 
 
 
 ##########################
 #
 # EventFeeder parameters
 #
 
 [eventfeeder]
 
 # number of threads
 nThreads = 3
 
 # max number of workers to try in one cycle
 maxWorkers = 500
 
 # lock interval in sec
 lockInterval = 600
 
 # sleep interval in sec
 sleepTime = 60
 
 
 
 
 ##########################
 #
 # Cacher parameters
 #
 
 [cacher]
 
 # one data ( main_key_name|sub_key_name|URL|dump_file(optional) ) following a white space per line
 #
 # Notes: This example is for five data. ddm_endpoints and panda_queues json files are retrieved using http.
 #        It also caches proxy files which are renewed by Credential Manager. Access key for BNL object store
 #        is retrieved from panda.
 data =
  ddmendpoints_objectstores.json||https://atlas-cric.cern.ch/api/atlas/ddmendpoint/query/?json&state=ACTIVE&site_state=ACTIVE&preset=dict&json_pretty=1&type[]=OS_LOGS&type[]=OS_ES
  panda_queues.json||https://atlas-cric.cern.ch/api/atlas/pandaqueue/query/?json
  agis_ddmendpoints.json||https://atlas-cric.cern.ch/api/atlas/ddmendpoint/query/list/?json&state=ACTIVE&site_state=ACTIVE&preset=dict&json_pretty=1
  proxy_pilot||file://path_to/FIXME_proxy_pilot
  proxy_production||file://path_to/FIXME_proxy_production
  resource_types.json||panda_server:get_resource_types
  job_statistics.json||panda_server:get_job_stats
  job_statistics_new.json||panda_server:get_job_stats_new
  worker_statistics.json||panda_server:get_worker_stats_from_panda
 # BNL_key||panda_cache:BNL_ObjectStoreKey.pub&BNL_ObjectStoreKey
 # globus_secret||panda_cache:GlobusClientID_1&GlobusRefreshToken_1
 
 # refresh interval in minint
 refreshInterval = 10
 
 # sleep interval in sec
 sleepTime = 60
 
 
 
 
 
 ##########################
 #
 # Payload interaction parameters
 #
 
 [payload_interaction]
 
 # worker attributes
 workerAttributesFile = worker_attributes.json
 
 # job report
 jobReportFile = jobReport.json
 
 # event status dump file in json
 eventStatusDumpJsonFile = event_status.dump.json
 
 # event status dump file in xml
 eventStatusDumpXmlFile = _event_status.dump
 
 # job request
 jobRequestFile = worker_requestjob.json
 
 # job spec file
 jobSpecFile = HPCJobs.json
 
 # event request
 eventRequestFile = worker_requestevents.json
 
 # event ranges file
 eventRangesFile = JobsEventRanges.json
 
 # update events
 updateEventsFile = worker_updateevents.json
 
 # PFC for input files
 xmlPoolCatalogFile = PoolFileCatalog_H.xml
 
 # get PandaIDs
 pandaIDsFile = worker_pandaids.json
 
 # request to be killed
 killWorkerFile = kill_worker.json
 
 # heartbeat from worker
 heartbeatFile = worker_heartbeat.json
 
 
 
 ##########################
 #
 # Front-end parameters
 #
 
 [frontend]
 
 # port number for simple http frontend. For apache frontend port number is set in httpd.conf
 portNumber = 25080
 
 # number of threads
 nThreads = 10
 
 # verbose
 verbose = False
 
 # type : simple or apache
 type = simple
 
 # enable token authentication of apache frontend; default is True
 authEnable = True
 
 # file of secret used in token signature
 secretFile = /FIXME
 
 # whether to verify token (of its signature, expiration, etc.) when decoding token
 verifyToken = True
 
 
 
 
 ##########################
 #
 # Sweeper parameters
 #
 
 [sweeper]
 
 # number of threads
 nThreads = 3
 
 # max number of workers to try in one cycle
 maxWorkers = 500
 
 # check interval in sec
 checkInterval = 180
 
 # sleep interval in sec
 sleepTime = 60
 
 # duration in hours to keep finished workers
 keepFinished = 24
 
 # duration in hours to keep failed workers
 keepFailed = 72
 
 # duration in hours to keep cancelled workers
 keepCancelled = 72
 
 # duration in hours to keep missed workers
 keepMissed = 24
 
 # disk cleaning interval in hours
 #diskCleanUpInterval = 1
 
 # comma-concatenated list of directory_name|high_watermark_in_GB to be cleaned up
 #diskHighWatermark = /dir1/subdir1|1000,/dir2/subdir2|5000
 
 
 
 ##########################
 #
 # Watcher parameters
 #
 
 [watcher]
 
 # a comma-concatenated list of file name of logs to watch (default: panda-db_proxy.log)
 logFileNameList = panda-db_proxy.log
 
 # action is taken when the last message is older than maxStalled sec. set 0 to disable the action
 maxStalled = 300
 
 # the number of messages to check interval
 nMessages = 1000
 
 # action is taken when it took more than maxDuration sec to generate nMessages messages. set 0 to disable the action
 maxDuration = 600
 
 # check interval in sec
 checkInterval = 180
 
 # sleep interval in sec
 sleepTime = 60
 
 # a comma-concatenated list of actions (email: to send alarms, kill: to kill forcefully, terminate: to kill with SIGTERM). or empty if no action
 actions =
 
 # name of env variable to keep pass-phrase
 passphraseEnv = HARVESTER_WATCHER_PASSPHRASE
 
 # hostname of SMTP server. note that parameters with the prefix of "mail" are required on
 mailServer = localhost
 
 # port of SMTP server
 mailPort = 25
 
 # use SSL for SMTP
 mailUseSSL = False
 
 # login user of SMTP server if any. leave it empty if SMTP doesn't need to logon
 mailUser =
 
 # login password of SMTP server if any. leave it empty if SMTP doesn't need to logon
 mailPassword =
 
 # email sender
 mailFrom = example_from@example.com
 
 # a comma-concatenated list of email recipients
 mailTo = example_to_1@example.com,example_to_2@example.com
 
 ##########################
 #
 # APF monitoring parameters
 #
 [apfmon]
 active = True
 
 ##########################
 #
 # Service monitor parameters
 #
 
 [service_monitor]
 active = True
 
 # optional in case you want to monitor any disk volume
 disk_volumes = data
 
 # pidfile only necessary when running in uwsgi
 pidfile = /var/log/harvester/panda_harvester.pid
 
 ##########################
 #
 # Google cloud parameters
 #
 
 [googlecloud]
 
 # zone where you are booting up your VMs and storage, e.g. us-east1-b
 zone = us-east1-b
 # project defined in the google compute account, where the activity will be billed
 project = atlas-harvester
 # private service account json generated in the google cloud management console
 service_account_file = /path/to/service_file.json
 # file with the user data to send to CERN VM
 user_data_file = /path/to/user_data.txt
 # image to use
 image = https://www.googleapis.com/compute/v1/projects/atlas-harvester/global/images/cernvm4-micro-3-0-6
 # harvester frontend
 harvester_frontend = aipanda170.cern.ch:25443
 
 
 
 
 ##########################
 #
 # File Syncer parameters
 #
 
 [file_syncer]
 
 # plugin configs in json
 # pluginConfigs =
 #     [
 #       {
 #         "module": "pandaharvester.harvesterfilesyncer.git_file_syncer",
 #         "name": "GitFileSyncer",
 #         "configs": {
 #           "grid_qconf": {
 #             "targetDir": "/data/atlpan/harvester_configurations",
 #             "sourceURL": "https://github.com/PanDAWMS/harvester_configurations.git",
 #             "sourceBranch": "master",
 #             "sourceRemoteName": "origin",
 #             "sourceSubdir": "GRID/condor_sdf_templates"
 #           }
 #         }
 #       }
 #     ]
 
 # sleep interval in sec
 sleepTime = 600
 
 
 
 ##########################
 #
 # Log level parameters
 #
 # To set logging level for each logger
 #
 # Notes: Global logging level is set by log_level in panda_common.cfg
 
 [log_level]
 
 # logger_name = level (CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET)
 #
 # E.g. to set INFO level to panda-monitor.log
 # monitor = INFO

This page was automatically generated by the 2.3.7 LXR engine. The LXR team