-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_helper.sh
executable file
·68 lines (47 loc) · 2.01 KB
/
run_helper.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
args="$@"
IFS=' ' read -r -a data_chunk <<< "$args"
echo "${#data_chunk[@]} data_chunk received!"
declare -a countries
for file in $(ls ./nordvpn_servers/ | shuf); # to minimize risk of more than one container connecting to the same NordVPN server at the same time
do
if [ "$file" != "servers.txt" ]; then
#echo "$file"
countries=(${countries[@]} "./nordvpn_servers/$file")
fi
done
echo "${#countries[@]} countries"
# Pass variables from .env
export $(egrep -v '^#' .env | xargs)
randint=$[RANDOM]
container_name="$randint"
echo "Creating a directory ./crawled_data/$randint..."
mkdir -p ./crawled_data/$randint
docker run --restart unless-stopped -ti --cap-add=NET_ADMIN --cap-add=SYS_MODULE --device /dev/net/tun --name $container_name \
--sysctl net.ipv4.conf.all.rp_filter=2 \
-e USER=$NORDVPN_USERNAME -e PASS=$NORDVPN_PASSWORD \
-e CONNECT="" -e TECHNOLOGY=NordLynx -d bubuntux/nordvpn
SLEEP_TIME=20
for data_point in "${data_chunk[@]}"
do
for country_file in "${countries[@]}"
do
echo "Sleeping for $SLEEP_TIME seconds..."
sleep $SLEEP_TIME
code=`sort --random-sort $country_file | head -n 1`
echo "Random code $code selected from $country_file..."
echo "Renaming $container_name container..."
docker rename $container_name "$code"_"$randint"
container_name="$code"_"$randint"
echo "Connecting to $code..."
docker exec $container_name nordvpn c $code
echo "Sleeping for $SLEEP_TIME seconds..."
sleep $SLEEP_TIME
echo "Spinning up a selenium container for location $code..."
docker run -v "$(pwd)"/crawled_data/$randint:/run_dir/data --net=container:$container_name --rm --name "selenium_$randint" selenium $data_point $code
done
done
echo "Logging out from the container $container_name..."
docker exec -it $container_name nordvpn logout
sleep 10
echo "Stopping and removing the container $container_name..."
docker stop $container_name && docker rm $container_name