tools/env/install_idds_full.sh

0001 #!/bin/bash
0002
0003 # as root
0004 yum install -y httpd.x86_64 conda gridsite mod_ssl.x86_64 httpd-devel.x86_64 gcc.x86_64 supervisor.noarch
0005 # yum install -y gfal2-plugin-gridftp gfal2-plugin-file.x86_64  gfal2-plugin-http.x86_64   gfal2-plugin-xrootd.x86_64  gfal2-python.x86_64 gfal2-python3.x86_64 gfal2-all.x86_64
0006 # conda install -c conda-forge python-gfal2
0007 # pip install requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle  unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad
0008
0009 mkdir /opt/idds
0010 mkdir /opt/idds_source
0011 mkdir /opt/idds
0012 mkdir /var/log/idds
0013 mkdir /var/log/idds/wsgisocks
0014 mkdir /tmp/idds/wsgisocks
0015 chown atlpilo1 -R /opt/idds
0016 chown atlpilo1 -R /opt/idds_source
0017 chown atlpilo1 /var/log/idds
0018 chown apache -R /var/log/idds/wsgisocks
0019 chown apache -R /tmp/idds/wsgisocks
0020
0021 cd /opt/idds_source
0022 #  rm -fr *; cp -r /afs/cern.ch/user/w/wguan/workdisk/iDDS/* .;python setup.py install --old-and-unmanageable
0023 git clone @github_idds@ /opt/idds_source
0024 conda env create --prefix=/opt/idds -f main/tools/env/environment.yml
0025 # source /etc/profile.d/conda.sh
0026 conda activate /opt/idds
0027 conda install -c conda-forge python-gfal2
0028
0029 pip install rucio-clients-atlas rucio-clients panda-client-light panda-client
0030 # root ca.crt to  /opt/idds/etc/ca.crt
0031
0032 pip install requests SQLAlchemy urllib3 retrying mod_wsgi flask futures stomp.py cx-Oracle  unittest2 pep8 flake8 pytest nose sphinx recommonmark sphinx-rtd-theme nevergrad
0033  pip install psycopg2-binary
0034
0035 # add "auth_type = x509_proxy" to /opt/idds/etc/rucio.cfg
0036
0037 # python setup.py install --old-and-unmanageable
0038 # Generate httpd conf .install_template files from the running Python env.
0039 # Run this after 'pip install' / conda install of the iDDS packages so that
0040 # site-packages / home / bin paths reflect the target machine, not the build env.
0041 python3 /opt/idds/tools/env/setup_httpd_conf.py
0042 # cp /opt/idds/etc/idds/rest/httpd-idds-443-py39-cc7.conf.install_template /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf
0043 # cp /opt/idds/etc/idds/rest/httpd-idds-443-py36-cc7.conf.install_template /etc/httpd/conf.d/httpd-idds-443-py36-cc7.conf
0044
0045 # scp wguan@aipanda102:/opt/idds/etc/rucio.cfg /opt/idds/etc/rucio.cfg
0046 # scp wguan@aipanda102:/etc/httpd/conf.d/httpd-idds-443-py310-al.conf /etc/httpd/conf.d/httpd-idds-443-py310-al.conf
0047 # scp wguan@aipanda102:/etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf /etc/httpd/conf.d/httpd-idds-443-py39-cc7.conf
0048 # mv /etc/httpd/conf.d/httpd-idds-443-py310-al.conf /etc/httpd/conf.d/httpd-idds-443-py310-al.conf.bac
0049 # scp wguan@aipanda102:/etc/supervisord.d/idds.ini /etc/supervisord.d/idds.ini
0050 # scp wguan@aipanda102:/opt/idds/etc/idds/idds.cfg /opt/idds/etc/idds/idds.cfg
0051 # scp wguan@aipanda102:/opt/idds/etc/idds/rest/gacl /opt/idds/etc/idds/rest/gacl
0052 # scp wguan@aipanda102:/opt/idds/etc/idds/auth/auth.cfg /opt/idds/etc/idds/auth/auth.cfg
0053
0054 chown atlpilo1 -R /opt/idds
0055 chown atlpilo1 -R /opt/idds_source
0056
0057 cp /opt/idds/etc/idds/idds.cfg.template /opt/idds/etc/idds/idds.cfg
0058
0059 # comment /etc/httpd/conf.d/ssl.conf "Listen 443 https"
0060 systemctl restart httpd.service
0061 systemctl enable httpd.service
0062
0063
0064 cp /opt/idds/etc/idds/supervisord.d/idds.ini /etc/supervisord.d/idds.ini
0065 cp /opt/idds_source/main/etc/idds/supervisord.d/idds.ini /etc/supervisord.d/idds.ini
0066
0067 systemctl start supervisord
0068 systemctl status supervisord
0069 systemctl enable supervisord
0070 #supervisorctl status
0071 #supervisorctl start all
0072 #supervisorctl stop all
0073
0074
0075 #condor
0076 yum install https://research.cs.wisc.edu/htcondor/repo/current/htcondor-release-current.el9.noarch.rpm
0077 # cp /etc/yum.repos.d/htcondor* /etc/yum-puppet.repos.d/
0078 # yum install -y condor.x86_64 condor-python.x86_64
0079 yum install -y condor.x86_64 python3-condor.x86_64
0080 #firewall-cmd --zone=public --add-port=9618/tcp --permanent
0081 firewall-cmd --zone=public --add-port=9618/udp --permanent
0082 firewall-cmd --zone=public --add-port=9600-9700/tcp --permanent
0083 firewall-cmd --reload
0084 cp /opt/idds_source/main/etc/condor/submitter/00personal_condor.config /etc/condor/config.d/
0085 systemctl enable condor
0086 systemctl start condor
0087 systemctl status condor
0088
0089
0090 #docker https://docs.docker.com/engine/install/linux-postinstall/
0091 groupadd docker
0092 yum install docker
0093 systemctl start docker
0094 systemctl status docker
0095 systemctl enable docker
0096 usermod -aG docker $(whoami)
0097 usermod -aG docker nobody  # for condor jobs which are running in this account
0098
0099 # shpinx https://sphinx-rtd-tutorial.readthedocs.io/en/latest/sphinx-config.html
0100 #[wguan@lxplus723 docs]$ make html
0101 pip install --upgrade sphinx
0102 pip install --upgrade sphinx-rtd-theme
0103 sphinx-quickstart
0104 make clean
0105 make html
0106 sphinx-apidoc -f -o ./source/codes/main/ ../main/lib/idds
0107 sphinx-apidoc -f -o ./source/codes/common/ ../common/lib/idds
0108 sphinx-apidoc -f -o ./source/codes/client/ ../client/lib/idds
0109 sphinx-apidoc -f -o ./source/codes/workflow/ ../workflow/lib/idds
0110 sphinx-apidoc -f -o ./source/codes/atlas/ ../atlas/lib/idds
0111 sphinx-apidoc -f -o ./source/codes/doma/ ../doma/lib/idds
0112
0113
0114 yum install fetch-crl.noarch
0115 yum install lcg-CA
0116
0117
0118 yum install redis
0119 systemctl start redis
0120 systemctl enable redis
0121
0122
0123 # install nats
0124
0125 yum install -y https://github.com/nats-io/nats-server/releases/download/v2.11.9/nats-server-v2.11.9-amd64.rpm https://github.com/nats-io/natscli/releases/download/v0.2.4/nats-0.2.4-amd64.rpm
0126 source /etc/profile.d/conda.sh; conda activate /opt/idds;
0127 pip install nats-py asyncio
0128
0129 nats --server nats://127.0.0.1:4222 --token my_default_token pub "event.UpdateProcessing" '{"foo":"bar"}'
0130
0131 nats --token my_default_token --server=127.0.0.1:4222 stream ls
0132 nats --server nats://127.0.0.1:4222 --token my_default_token stream ls
0133 nats --server nats://127.0.0.1:4222 --token my_default_token stream info event_stream
0134 nats --server nats://127.0.0.1:4222 --token my_default_token stream get event_stream --last-for=event.UpdateProcessing
0135 nats --server nats://127.0.0.1:4222 --token my_default_token consumer list event_stream
0136