Skip to content
This repository was archived by the owner on Dec 23, 2024. It is now read-only.

allow to specify thread counts (sync,build) #201

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

allow to specify thread counts (sync,build) #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Scripts/Common/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export -f startPatcher;

resetWorkspace() {
umask 0022;
repo forall -c 'git add -A && git reset --hard' && rm -rf out DOS_PATCHED_FLAG && repo sync -j8 --force-sync --detach;
repo forall -j${DOS_MAX_THREADS_REPO} -c 'git add -A && git reset --hard' && rm -rf out DOS_PATCHED_FLAG && repo sync -j${DOS_MAX_THREADS_REPO} --force-sync --detach;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

using too many threads on sync will get you rate limited by google and temporarily blocked

using many threads on reset may cause thrashing, and already completes quickly

Copy link
Contributor Author

@steadfasterX steadfasterX Apr 2, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's why there's a max allowed value which avoids being rate limited. It uses your current used value.

Again this is more to limit (e.g using 1 helps for debug but using other values can help speeding up the process - yes even for reset)

}
export -f resetWorkspace;

Expand All @@ -33,7 +33,7 @@ patchWorkspace() {
export -f patchWorkspace;

verifyAllPlatformTags() {
repo forall -v -c 'sh -c "source $DOS_WORKSPACE_ROOT/Scripts/Common/Tag_Verifier.sh && verifyTagIfPlatform $REPO_PROJECT $REPO_PATH"';
repo forall -j${DOS_MAX_THREADS_REPO} -v -c 'sh -c "source $DOS_WORKSPACE_ROOT/Scripts/Common/Tag_Verifier.sh && verifyTagIfPlatform $REPO_PROJECT $REPO_PATH"';
}
export -f verifyAllPlatformTags;

Expand Down
4 changes: 2 additions & 2 deletions Scripts/LineageOS-14.1/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ buildDevice() {
cd "$DOS_BUILD_BASE";
export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
pkill java && sleep 10; #XXX: ugly hack
breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
Copy link
Member

@SkewedZeppelin SkewedZeppelin Apr 1, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mka already defaults to all cores +2

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes and this change allows it to decrease it. It's not needed when you want to use all cores but this not always wanted - at least for me

pkill java && sleep 10; #XXX: ugly hack
}
export -f buildDevice;
Expand All @@ -43,7 +43,7 @@ buildDeviceUserDebug() {
cd "$DOS_BUILD_BASE";
if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then
export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
breakfast "lineage_$1-userdebug" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-userdebug" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
else
echo -e "\e[0;31mNo signing keys available for $1\e[0m";
fi;
Expand Down
2 changes: 1 addition & 1 deletion Scripts/LineageOS-15.1/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ buildDevice() {
cd "$DOS_BUILD_BASE";
if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then
export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
else
echo -e "\e[0;31mNo signing keys available for $1\e[0m";
fi;
Expand Down
2 changes: 1 addition & 1 deletion Scripts/LineageOS-16.0/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ buildDevice() {
cd "$DOS_BUILD_BASE";
if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then
export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
else
echo -e "\e[0;31mNo signing keys available for $1\e[0m";
fi;
Expand Down
2 changes: 1 addition & 1 deletion Scripts/LineageOS-17.1/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ buildDevice() {
cd "$DOS_BUILD_BASE";
if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then
export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
else
echo -e "\e[0;31mNo signing keys available for $1\e[0m";
fi;
Expand Down
2 changes: 1 addition & 1 deletion Scripts/LineageOS-18.1/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ buildDevice() {
cd "$DOS_BUILD_BASE";
if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then
export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
else
echo -e "\e[0;31mNo signing keys available for $1\e[0m";
fi;
Expand Down
2 changes: 1 addition & 1 deletion Scripts/LineageOS-19.1/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ buildDevice() {
cd "$DOS_BUILD_BASE";
if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then
export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
else
echo -e "\e[0;31mNo signing keys available for $1\e[0m";
fi;
Expand Down
2 changes: 1 addition & 1 deletion Scripts/LineageOS-20.0/Functions.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ buildDevice() {
cd "$DOS_BUILD_BASE";
if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then
#export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1";
breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2;
breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2;
else
echo -e "\e[0;31mNo signing keys available for $1\e[0m";
fi;
Expand Down
41 changes: 41 additions & 0 deletions Scripts/init.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
#START OF USER CONFIGURABLE OPTIONS
#
#General
#export DOS_MAX_THREADS_REPO=8; #Max amount of threads used for sync, e.g. used by repo (shouldn't exceed your CPU cores count, due to rate limit restrictions this cannot be set higher then 8)
#export DOS_MAX_THREADS_BUILD="nolimit"; #Max amount of threads used for build, e.g. used by make (shouldn't exceed your CPU cores count)
export DOS_WORKSPACE_ROOT="/mnt/dos/"; #XXX: THIS MUST BE CORRECT TO BUILD!
#export DOS_BUILDS=$DOS_WORKSPACE_ROOT"Builds/";
export DOS_BUILDS="/mnt/Drive-4/DOS/Builds/"; #XXX: THIS MUST BE CORRECT TO BUILD!
Expand Down Expand Up @@ -109,6 +111,45 @@ export DOS_THEME_700="E64A19"; #Deep Orange 700

umask 0022;

# by default we will calculate the max CPU count automatically (used by e.g. repo commands)
# if the required tool "nproc" is not found a default is used.
export FALLBACK_MAX_THREADS_REPO=4;
export MAX_THREADS_REPO_RATE=8; #to avoid being rate limited we never go above this for syncing
export FALLBACK_MAX_THREADS_BUILD="nolimit"; #if nothing specified we will use all CPU power available

calcThreads(){
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is needless complexity

Copy link
Contributor Author

@steadfasterX steadfasterX Apr 2, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see any needless conplexity here but more flexibility. You might not need it which is ok but i do (and maybe I'm not the only one) ;)

That being said I'm fine if you wanna close this pr

unset _MAX_THREADS
DEFAULT_COUNT=$1
nproc --version > /dev/null 2>&1
if [ $? -eq 10 ];then
_MAX_THREADS_ALL=$(nproc --all)
_MAX_THREADS=$(( _MAX_THREADS_ALL - 1))
fi
if [ -z "$_MAX_THREADS" ];then
echo $DEFAULT_COUNT
return 9
else
echo $_MAX_THREADS
fi
}
export -f calcThreads;

if [ -z "$DOS_MAX_THREADS_REPO" ];then
export DOS_MAX_THREADS_REPO=$(calcThreads $FALLBACK_MAX_THREADS_REPO) \
|| echo -e "\e[0;33mWARNING: could not calculate thread count and no user defined amount specified. This could slow down the SYNC processes.\e[0m"
fi
if [ "$DOS_MAX_THREADS_REPO" -gt "$MAX_THREADS_REPO_RATE" ];then
echo -e "\e[0;33mWARNING: Your specified DOS_MAX_THREADS_REPO value ($DOS_MAX_THREADS_REPO) could raise rate limits so has been decreased to >$MAX_THREADS_REPO_RATE<.\e[0m"
export DOS_MAX_THREADS_REPO=$MAX_THREADS_REPO_RATE
fi
if [ -z "$DOS_MAX_THREADS_BUILD" ];then
export DOS_MAX_THREADS_BUILD=$(calcThreads $FALLBACK_MAX_THREADS_BUILD)\
|| echo -e "\e[0;33mWARNING: could not calculate thread count and no user defined amount specified. This could slow down the BUILD processes.\e[0m"
fi
echo "Activated CPU count (sync): $DOS_MAX_THREADS_REPO"
echo "Activated CPU count (build): $DOS_MAX_THREADS_BUILD"
[ "$DOS_MAX_THREADS_BUILD" == "nolimit" ] && export DOS_MAX_THREADS_BUILD="" #actually means "all" when empty

gpgVerifyGitHead() {
if [ -r "$DOS_TMP_GNUPG/pubring.kbx" ]; then
if git -C "$1" verify-commit HEAD &>/dev/null; then
Expand Down