Skip to content

Commit cd54667

Browse files
author
haoquan
committed
_
2 parents 0379597 + 414386a commit cd54667

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+4489320
-1
lines changed

‎.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ __pycache__/
55
*.py[cod]
66
*$py.class
77

8-
*.txt
98
*.pt
109
*.zip
1110

@@ -103,4 +102,7 @@ ENV/
103102
# mkdocs documentation
104103
/site
105104
checkpoints/
105+
<<<<<<< HEAD
106106
checkpoint/
107+
=======
108+
>>>>>>> 414386a23b44dac0ee97178f5b62cf17e066f22e

‎README.md

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# 环境初始化
2+
3+
修改init.sh中dataset所在目录
4+
5+
```
6+
# #################### link dataset
7+
DATASET_ROOT=/home/tione/notebook/algo-2021/dataset/
8+
```
9+
10+
执行初始化
11+
12+
`sudo chmod a+x ./init.sh && ./init.sh run`
13+
14+
# 训练
15+
`sudo chmod a+x ./run.sh && ./run.sh`
16+
`sudo chmod a+x ./run2.sh && ./run2.sh`

‎config/temp.txt

Whitespace-only changes.

‎dataset

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/home/tione/notebook/algo-2021/dataset/

‎init_.sh

+9
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ if [ "${ACTION}" != "run" ]; then
5252
exit 0
5353
fi
5454

55+
# #################### link dataset
56+
echo ["Link dataset"]
57+
DATASET_ROOT=/home/tione/notebook/algo-2021/dataset/
58+
ln -s ${DATASET_ROOT} ${CODE_ROOT}
59+
5560
# #################### install system libraries
5661
if [ "${OS_ID}" == "ubuntu" ]; then
5762
echo "[Info] installing system libraries in ${OS_ID}"
@@ -102,6 +107,7 @@ python -m ipykernel install --user --name ${CONDA_NEW_ENV} --display-name "TAAC2
102107

103108
# #################### install python libraries
104109
pip install torch==1.7.1+cu101 torchvision==0.8.2+cu101 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
110+
<<<<<<< HEAD:init_.sh
105111

106112
# bert
107113
echo "[Bert install]"
@@ -130,6 +136,9 @@ cd -
130136
# nextvlad
131137
pip install timm
132138

139+
=======
140+
pip install transformers opencv-python
141+
>>>>>>> 414386a23b44dac0ee97178f5b62cf17e066f22e:init.sh
133142

134143
# check tensorflow GPU
135144
# python -c "import torch; print(torch.__version__)"

‎pretrain_models/temp.txt

Whitespace-only changes.

‎run.sh

100644100755
File mode changed.

‎run2.sh

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env bash
2+
3+
# #################### get env directories
4+
# CONDA_ROOT
5+
CONDA_CONFIG_ROOT_PREFIX=$(conda config --show root_prefix)
6+
echo "CONDA_CONFIG_ROOT_PREFIX= ${CONDA_CONFIG_ROOT_PREFIX}"
7+
get_conda_root_prefix() {
8+
TMP_POS=$(awk -v a="${CONDA_CONFIG_ROOT_PREFIX}" -v b="/" 'BEGIN{print index(a, b)}')
9+
TMP_POS=$((TMP_POS-1))
10+
if [ $TMP_POS -ge 0 ]; then
11+
echo "${CONDA_CONFIG_ROOT_PREFIX:${TMP_POS}}"
12+
else
13+
echo ""
14+
fi
15+
}
16+
CONDA_ROOT=$(get_conda_root_prefix)
17+
if [ ! -d "${CONDA_ROOT}" ]; then
18+
echo "CONDA_ROOT= ${CONDA_ROOT}, not exists, exit"
19+
exit 1
20+
fi
21+
# CONDA ENV
22+
CONDA_NEW_ENV=taac2021-tagging-jbtjjsw
23+
# JUPYTER_ROOT
24+
JUPYTER_ROOT=/home/tione/notebook
25+
if [ ! -d "${JUPYTER_ROOT}" ]; then
26+
echo "JUPYTER_ROOT= ${JUPYTER_ROOT}, not exists, exit"
27+
exit 1
28+
fi
29+
# CODE ROOT
30+
CODE_ROOT=${JUPYTER_ROOT}/algo-2021-jbtjjsw
31+
if [ ! -d "${CODE_ROOT}" ]; then
32+
echo "CODE_ROOT= ${CODE_ROOT}, not exists, exit"
33+
exit 1
34+
fi
35+
# DATASET ROOT
36+
echo ["Link dataset"]
37+
DATASET_ROOT=/home/tione/notebook/algo-2021/dataset/
38+
ln -s ${DATASET_ROOT} ${CODE_ROOT}
39+
40+
DATASET_ROOT=${CODE_ROOT}/dataset
41+
if [ ! -d "${DATASET_ROOT}" ]; then
42+
echo "DATASET_ROOT= ${DATASET_ROOT}, not exists, exit"
43+
exit 1
44+
fi
45+
# OS RELEASE
46+
OS_ID=$(awk -F= '$1=="ID" { print $2 ;}' /etc/os-release)
47+
48+
echo "CONDA_ROOT= ${CONDA_ROOT}"
49+
echo "CONDA_NEW_ENV= ${CONDA_NEW_ENV}"
50+
echo "JUPYTER_ROOT= ${JUPYTER_ROOT}"
51+
echo "CODE_ROOT= ${CODE_ROOT}"
52+
echo "DATASET_ROOT= ${DATASET_ROOT}"
53+
echo "OS_ID= ${OS_ID}"
54+
55+
# #################### activate conda env and check lib versions
56+
# solve run problem in Jupyter Notebook
57+
# conda in shell propagation issue - https://stackoverflow.com/questions/52779016/conda-command-working-in-command-prompt-but-not-in-bash-script/52813960#52813960
58+
CONDA_CONFIG_FILE="${CONDA_ROOT}/etc/profile.d/conda.sh"
59+
if [ ! -f "${CONDA_CONFIG_FILE}" ]; then
60+
echo "CONDA_CONFIG_FILE= ${CONDA_CONFIG_FILE}, not exists, exit"
61+
exit 1
62+
fi
63+
# shellcheck disable=SC1090
64+
source "${CONDA_CONFIG_FILE}"
65+
66+
# ###### activate conda env
67+
# conda env by name
68+
# conda activate ${CONDA_NEW_ENV}
69+
# conda env by prefix
70+
conda activate ${JUPYTER_ROOT}/envs/${CONDA_NEW_ENV}
71+
conda info --envs
72+
73+
# check library versions
74+
echo "[Pytorch version]"
75+
python -c "import torch; print(torch.__version__)"
76+
echo "[Cuda available]"
77+
python -c "import torch; print(torch.cuda.is_available())"
78+
echo "[NumPy]"
79+
python -c "import numpy as np; print(np.__version__)"
80+
echo "[OpenCV]"
81+
python -c "import cv2; print(cv2.__version__)"
82+
echo "[Transformers]"
83+
python -c "import transformers; print(transformers.__version__)"
84+
85+
# #################### get 1st input argument as TYPE
86+
TYPE=train
87+
if [ -z "$1" ]; then
88+
echo "[Warning] TYPE is not set, using 'train' as default"
89+
else
90+
TYPE=$(echo "$1" | tr '[:upper:]' '[:lower:]')
91+
echo "[Info] TYPE is ${TYPE}"
92+
fi
93+
94+
# #################### execute according to TYPE
95+
########## train
96+
if [ "$TYPE" = "train" ]; then
97+
# cd src/text_predict/ocr_bert_base
98+
# time python ocr_classifier.py
99+
# cd -
100+
cd src/text_predict/ocr_bert_class_train
101+
# time python train.py
102+
cd -
103+
104+
cd src/weight_fusion
105+
time python train.py
106+
cd -
107+
exit 0
108+
109+
########## test
110+
elif [ "$TYPE" = "test" ]; then
111+
cd src/text_predict/ocr_bert_class_train
112+
time python predict.py
113+
cd -
114+
cd src/weight_fusion
115+
time python predict.py
116+
cd -
117+
exit 0
118+
# ######### text predict
119+
fi

0 commit comments

Comments
 (0)