main/config_default/idds.cfg

0001 # Licensed under the Apache License, Version 2.0 (the "License");
0002 # You may not use this file except in compliance with the License.
0003 # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
0004 #
0005 # Authors:
0006 # - Wen Guan, <wen.guan@cern.ch>, 2019 - 2023
0007
0008 [common]
0009 #logdir = /var/log/idds
0010 # loglevel = DEBUG
0011 loglevel = INFO
0012 num_threads = 16
0013
0014 [database]
0015 default = sqlite:////tmp/idds.db
0016 pool_size=10
0017 pool_recycle=3600
0018 echo=0
0019 pool_reset_on_return=rollback
0020
0021 [rest]
0022 host = https://localhost:443/idds
0023 cacher_dir = /var/log/idds
0024
0025 [main]
0026 # agents = clerk, transformer, carrier, conductor
0027 # agents = clerk, transformer, submitter, poller, receiver, trigger, finisher, conductor
0028 # agents = clerk, transformer, submitter, poller, receiver, trigger, finisher, conductor, archiver, coordinator
0029 agents = clerk, transformer, submitter, poller, receiver, trigger, finisher, conductor, archiver, natscoordinator, transceiver
0030
0031 [eventbus]
0032 # backend = database
0033 # backend = message
0034 backend = default
0035 # debug = True
0036
0037 [coordinator]
0038 coordination_interval_delay = 300
0039
0040
0041 [clerk]
0042 num_threads = 8
0043 max_number_workers = 8
0044 poll_period = 300
0045 new_poll_period = 10
0046 update_poll_period = 300
0047 new_command_poll_period = 10
0048 update_command_poll_period = 300
0049 max_update_poll_period = 300
0050 max_new_poll_period = 300
0051
0052 poll_time_period = 60
0053 poll_operation_time_period = 60
0054 retrieve_bulk_size = 32
0055 pending_time = 30
0056
0057 [transformer]
0058 num_threads = 16
0059 poll_period = 180
0060 new_poll_period = 10
0061 update_poll_period = 180
0062
0063 max_update_poll_period = 300
0064 max_new_poll_period = 300
0065
0066 poll_time_period = 60
0067 retrieve_bulk_size = 64
0068 poll_operation_time_period = 180
0069 message_bulk_size = 1000
0070
0071 # domapandawork.life_time = 86400
0072 domapandawork.num_retries = 0
0073
0074 [carrier]
0075 num_threads = 16
0076 max_number_workers = 16
0077 trigger_max_number_workers = 20
0078 finisher_max_number_workers = 16
0079 receiver_num_threads = 16
0080
0081 poll_period = 60
0082 new_poll_period = 10
0083 update_poll_period = 60
0084 update_poll_period_for_new_task = 120
0085
0086 max_update_poll_period = 300
0087 max_new_poll_period = 300
0088
0089 poll_time_period = 60
0090 poll_operation_time_period = 180
0091 retrieve_bulk_size = 16
0092 message_bulk_size = 1000
0093
0094 plugin.receiver = idds.agents.common.plugins.messaging.MessagingReceiver
0095 plugin.receiver.brokers = atlas-mb.cern.ch
0096 plugin.receiver.port = 61013
0097 # plugin.notifier.vhost =
0098 plugin.receiver.destination = /topic/doma.panda_idds
0099 plugin.receiver.username = user
0100 plugin.receiver.password = password
0101 plugin.receiver.broker_timeout = 10
0102
0103 plugin.receiver.channels = {"default": {"brokers": ["atlas-mb.cern.ch:61013"],
0104                                         "destination": "/topic/doma.panda_idds",
0105                                         "username": "user",
0106                                         "password": "password",
0107                                         "broker_timeout": 360}}
0108
0109 # domapandawork.life_time = 86400
0110 domapandawork.num_retries = 0
0111 domapandawork.poll_panda_jobs_chunk_size = 2000
0112
0113 plugin.iwork_submitter = idds.agents.carrier.plugins.panda.PandaSubmitterPoller
0114 plugin.iworkflow_submitter = idds.agents.carrier.plugins.panda.PandaSubmitterPoller
0115 plugin.iworkflow_poller = idds.agents.carrier.plugins.panda.PandaSubmitterPoller
0116 plugin.iwork_poller = idds.agents.carrier.plugins.panda.PandaSubmitterPoller
0117
0118 [conductor]
0119 delay = 120
0120 replay_times = 2
0121
0122 threshold_to_release_messages = 1000
0123 random_delay = 60
0124
0125 plugin.notifier = idds.agents.conductor.plugins.messaging.MessagingSender
0126 plugin.notifier.brokers = atlas-test-mb.cern.ch
0127 plugin.notifier.port = 61013
0128 plugin.notifier.destination = /queue/atlas.idds
0129 plugin.notifier.username = user
0130 plugin.notifier.password = password
0131 plugin.notifier.broker_timeout = 10
0132
0133 plugin.notifier.channels = {"default": {"brokers": ["atlas-mb.cern.ch:61013"],
0134                                         "destination": "/topic/doma.idds",
0135                                         "username": "user",
0136                                         "password": "password",
0137                                         "broker_timeout": 360},
0138                             "ContentExt": {"brokers": ["atlas-test-mb.cern.ch:61013"],
0139                                            "destination": "/queue/atlas.idds",
0140                                            "username": "user",
0141                                            "password": "password",
0142                                            "broker_timeout": 360}
0143                            }
0144
0145 [archiver]
0146 # days
0147 older_than = 60
0148 poll_period = 1
0149
0150 [asyncresult]
0151 broker_type = activemq
0152 brokers = atlas-test-mb.cern.ch:61013
0153 broker_destination = /topic/doma.idds
0154 broker_timeout = 360
0155 broker_username = domaidds
0156 broker_password = password
0157
0158 [prompt]
0159 namespace = dev
0160 num_threads = 4
0161 timetolive = 7200000
0162
0163 panda.num_workers = 1
0164 panda.vo = wlcg
0165 panda.queue = PANDA_EIC
0166 panda.site = BNL
0167 panda.cloud = US
0168 panda.working_group = EIC
0169 panda.priority = 500
0170 panda.core_count = 10
0171 panda.ram_count = 4000   # per core
0172 panda.max_walltime = 3600
0173 panda.max_attempt = 3
0174
0175 # worker related brokers
0176 worker_publisher_broker = {
0177     "broker": "localhost:61612",
0178     "destination": "/topic/panda.workers",
0179     "username": "admin",
0180     "password": "admin",
0181     "broker_timeout": 360
0182 }
0183
0184 worker_subscriber_broker = {
0185     "broker": "localhost:61612",
0186     "destination": "/topic/panda.workers",
0187     "username": "admin",
0188     "password": "admin",
0189     "broker_timeout": 360
0190 }
0191
0192 transformer_broadcast_broker = {
0193     "broker": "localhost:61612",
0194     "destination": "/topic/panda.transformer",
0195     "username": "admin",
0196     "password": "admin",
0197     "broker_timeout": 360
0198 }
0199
0200 # slice related brokers (for idds)
0201 slice_idds_subscriber_broker = {
0202     "broker": "localhost:61612",
0203     "destination": "/queue/panda.idds.slices",
0204     "username": "admin",
0205     "password": "admin",
0206     "broker_timeout": 360
0207 }
0208 result_idds_subscriber_broker = {
0209     "broker": "localhost:61612",
0210     "destination": "/queue/panda.idds.results",
0211     "username": "admin",
0212     "password": "admin",
0213     "broker_timeout": 360
0214 }
0215
0216 # slice related brokers (for transformer)
0217 slice_transformer_subscriber_broker = {
0218     "broker": "localhost:61612",
0219     "destination": "/queue/panda.transformer.slices",
0220     "username": "admin",
0221     "password": "admin",
0222     "broker_timeout": 360
0223 }
0224 result_publisher_broker = {
0225     "broker": "localhost:61612",
0226     "destination": "/topic/panda.results",
0227     "username": "admin",
0228     "password": "admin",
0229     "broker_timeout": 360
0230 }