forked from FirmwareDS/Capstone-Information-Retrieval-2.0
-
Notifications
You must be signed in to change notification settings - Fork 0
/
bash.sh
77 lines (53 loc) · 1.84 KB
/
bash.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
mkdir project
cd project
git clone https://github.com/FirmwareDS/Capstone-Information-Retrieval-2.0
#chmod 400 *.pem
cd Neural-Information-Retrieval/
source activate pytorch
sudo apt-get update
sudo apt-get install git-lfs
pip install transformers
pip install omegaconf
pip install hydra-core
pip install h5py
pip install pytrec_eval
pip install tensorboard
cd splade
wget https://download.europe.naverlabs.com/splade/sigir22/data.tar.gz
tar -xzvf data.tar.gz
rm data.tar.gz
cd weights
git clone https://huggingface.co/naver/splade-cocondenser-ensembledistil
git clone https://huggingface.co/naver/efficient-splade-VI-BT-large-query
git clone https://huggingface.co/naver/efficient-splade-VI-BT-large-doc
git clone https://huggingface.co/naver/efficient-splade-V-large-query
git clone https://huggingface.co/naver/efficient-splade-V-large-doc
cd ..
wget https://www.dropbox.com/s/hir60b9yj194dv7/mlm_flops.tar.gz?dl=0
tar -xzvf mlm_flops.tar.gz?dl=0
#jupyter setup
jupyter notebook password
cd ~
mkdir ssl
cd ssl
openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout mykey.key -out mycert.pem
cd ..
cd project
screen -S jupyter
#screen -ls
#screen -r 1234
jupyter notebook --certfile=~/ssl/mycert.pem --keyfile ~/ssl/mykey.key
#Ctral a + d
#local
#ssh -i ~/mykeypair.pem -N -f -L 8889:localhost:8888 ubuntu@ec2-54-219-129-71.us-west-1.compute.amazonaws.com
#training
screen -S train
export SPLADE_CONFIG_NAME="config_splade++_cocondenser_ensembledistil_monogpu.yaml"
python3 -m splade.all
# \ config.checkpoint_dir=monogpu/debug/checkpoint \
# config.index_dir=monogpu/debug/index \
# config.out_dir=experiments/debug/out
nvidia-smi
tensorboard --logdir=tensorboard
ssh -i w210.pem -N -f -L 6006:localhost:6006 ubuntu@ec2-13-56-212-101.us-west-1.compute.amazonaws.com
export SPLADE_CONFIG_FULLPATH=../conf/efficient_splade/config_BT_VI_large.yaml