import os, sys, pprint, re, json, pathlib, hashlib, subprocess, glob executePath = os.getcwd() scriptPath = os.path.dirname(os.path.realpath(__file__)) def finish(code): global executePath os.chdir(executePath) sys.exit(code) def error(text): print('[ERROR] ' + text) finish(1) arm64 = False arm = False x86 = False x86_64 = False ndkPath = "" win = (sys.platform == 'win32') mac = (sys.platform == 'darwin') dirSep = '\\' if win else '/' pathSep = ';' if win else ':' keysLoc = 'cache_keys' rootDir = os.getcwd() projDir = rootDir; print("PROJ", projDir) # usedPrefix = libsDir + dirSep + 'local' optionsList = [ 'skip-release', 'build-qt5', 'skip-qt5', 'build-qt6', 'skip-qt6', 'build-stackwalk', ] arches = [] options = [] runCommand = [] customRunCommand = False for arg in sys.argv[1:]: if arg == "arm64": arm64 = True arches.append(arg) options.append(arg) if arg == "arm": arm = True arches.append(arg) options.append(arg) if arg == "x86": x86 = True arches.append(arg) options.append(arg) if arg == "x86_64": x86_64 = True arches.append(arg) options.append(arg) if arg.startswith("ndk="): ndkPath = arg.split("ndk=")[1] options.append(arg) if customRunCommand: runCommand.append(arg) if arg in optionsList: options.append(arg) elif arg == 'run': customRunCommand = True if not os.path.isdir(projDir + '/' + keysLoc): pathlib.Path(projDir + '/' + keysLoc).mkdir(parents=True, exist_ok=True) # if not os.path.isdir(libsDir + '/' + keysLoc): # pathlib.Path(libsDir + '/' + keysLoc).mkdir(parents=True, exist_ok=True) # if not os.path.isdir(thirdPartyDir + '/' + keysLoc): # pathlib.Path(thirdPartyDir + '/' + keysLoc).mkdir(parents=True, exist_ok=True) environment = { 'MAKE_THREADS_CNT': '-j8', 'MACOSX_DEPLOYMENT_TARGET': '10.12', 'UNGUARDED': '-Werror=unguarded-availability-new', 'MIN_VER': '-mmacosx-version-min=10.12', 'PROJ_DIR': projDir, } ignoreInCacheForThirdParty = [ 'USED_PREFIX', 'LIBS_DIR', 'SPECIAL_TARGET', 'X8664', 'WIN32X64', ] environmentKeyString = '' envForThirdPartyKeyString = '' for key in environment: part = key + '=' + environment[key] + ';' environmentKeyString += part if not key in ignoreInCacheForThirdParty: envForThirdPartyKeyString += part environmentKey = hashlib.sha1(environmentKeyString.encode('utf-8')).hexdigest() envForThirdPartyKey = hashlib.sha1(envForThirdPartyKeyString.encode('utf-8')).hexdigest() modifiedEnv = os.environ.copy() for key in environment: modifiedEnv[key] = environment[key] def computeFileHash(path): sha1 = hashlib.sha1() with open(path, 'rb') as f: while True: data = f.read(256 * 1024) if not data: break sha1.update(data) return sha1.hexdigest() def computeCacheKey(stage): if (stage['location'] == 'ThirdParty'): envKey = envForThirdPartyKey else: envKey = environmentKey objects = [ envKey, stage['location'], stage['name'], stage['version'], stage['commands'] ] for pattern in stage['dependencies']: pathlist = glob.glob(libsDir + '/' + pattern) items = [pattern] if len(pathlist) == 0: pathlist = glob.glob(thirdPartyDir + '/' + pattern) if len(pathlist) == 0: error('Nothing found: ' + pattern) for path in pathlist: if not os.path.exists(path): error('Not found: ' + path) items.append(computeFileHash(path)) objects.append(':'.join(items)) return hashlib.sha1(';'.join(objects).encode('utf-8')).hexdigest() def keyPath(stage): return stage['directory'] + '/' + keysLoc + '/' + stage['name'] def checkCacheKey(stage): if not 'key' in stage: error('Key not set in stage: ' + stage['name']) key = keyPath(stage) if not os.path.exists(stage['directory'] + '/' + stage['name']): return 'NotFound' if not os.path.exists(key): return 'Stale' with open(key, 'r') as file: return 'Good' if (file.read() == stage['key']) else 'Stale' def clearCacheKey(stage): key = keyPath(stage) if os.path.exists(key): os.remove(key) def writeCacheKey(stage): if not 'key' in stage: error('Key not set in stage: ' + stage['name']) key = keyPath(stage) with open(key, 'w') as file: file.write(stage['key']) stages = [] def removeDir(folder): if win: return 'if exist ' + folder + ' rmdir /Q /S ' + folder + '\nif exist ' + folder + ' exit /b 1' return 'rm -rf ' + folder def filterByPlatform(commands): commands = commands.split('\n') result = '' dependencies = [] version = '0' skip = False for command in commands: m = re.match(r'(!?)([a-z0-9_]+):', command) if m and m.group(2) != 'depends' and m.group(2) != 'version': scopes = m.group(2).split('_') inscope = 'common' in scopes if arm64 and 'arm64' in scopes: inscope = True if arm and 'arm' in scopes: inscope = True if x86 and 'x86' in scopes: inscope = True if x86_64 and 'x86_64' in scopes: inscope = True # if linux and 'linux' in scopes: # inscope = True # if 'release' in scopes: # if 'skip-release' in options: # inscope = False # elif len(scopes) == 1: # continue skip = inscope if m.group(1) == '!' else not inscope elif not skip and not re.match(r'\s*#', command): if m and m.group(2) == 'version': version = version + '.' + command[len(m.group(0)):].strip() elif m and m.group(2) == 'depends': pattern = command[len(m.group(0)):].strip() dependencies.append(pattern) else: command = command.strip() if len(command) > 0: result = result + command + '\n' return [result, dependencies, version] def stage(name, commands, location = '.'): directory = projDir # if location == 'Libraries': # directory = libsDir # elif location == 'ThirdParty': # directory = thirdPartyDir # else: # error('Unknown location: ' + location) [commands, dependencies, version] = filterByPlatform(commands) if len(commands) > 0: stages.append({ 'name': name, 'location': location, 'directory': directory, 'commands': commands, 'version': version, 'dependencies': dependencies }) def winFailOnEach(command): commands = command.split('\n') result = '' startingCommand = True for command in commands: command = re.sub(r'\$([A-Za-z0-9_]+)', r'%\1%', command) if re.search(r'\$', command): error('Bad command: ' + command) appendCall = startingCommand and not re.match(r'(if|for) ', command) called = 'call ' + command if appendCall else command result = result + called if command.endswith('^'): startingCommand = False else: startingCommand = True result = result + '\r\nif %errorlevel% neq 0 exit /b %errorlevel%\r\n' return result def printCommands(commands): print('---------------------------------COMMANDS-LIST----------------------------------') print(commands, end='') print('--------------------------------------------------------------------------------') def run(commands): printCommands(commands) if win: if os.path.exists("command.bat"): os.remove("command.bat") with open("command.bat", 'w') as file: file.write('@echo OFF\r\n' + winFailOnEach(commands)) result = subprocess.run("command.bat", shell=True, env=modifiedEnv).returncode == 0 if result and os.path.exists("command.bat"): os.remove("command.bat") return result elif re.search(r'\%', commands): error('Bad command: ' + commands) else: return subprocess.run("set -e\n" + commands, shell=True, executable="/bin/bash", env=modifiedEnv).returncode == 0 # Thanks https://stackoverflow.com/a/510364 class _Getch: """Gets a single character from standard input. Does not echo to the screen.""" def __init__(self): try: self.impl = _GetchWindows() except ImportError: self.impl = _GetchUnix() def __call__(self): return self.impl() class _GetchUnix: def __init__(self): import tty, sys def __call__(self): import sys, tty, termios fd = sys.stdin.fileno() old_settings = termios.tcgetattr(fd) try: tty.setraw(sys.stdin.fileno()) ch = sys.stdin.read(1) finally: termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) return ch class _GetchWindows: def __init__(self): import msvcrt def __call__(self): import msvcrt return msvcrt.getch().decode('ascii') getch = _Getch() def runStages(): onlyStages = [] rebuildStale = False for arg in sys.argv[1:]: if arg in options: continue elif arg == 'silent': rebuildStale = True continue found = False for stage in stages: if stage['name'] == arg: onlyStages.append(arg) found = True break if not found: error('Unknown argument: ' + arg) count = len(stages) index = 0 for stage in stages: if len(onlyStages) > 0 and not stage['name'] in onlyStages: continue index = index + 1 version = ('#' + str(stage['version'])) if (stage['version'] != '0') else '' prefix = '[' + str(index) + '/' + str(count) + '](' + stage['location'] + '/' + stage['name'] + version + ')' print(prefix + ': ', end = '', flush=True) stage['key'] = computeCacheKey(stage) commands = removeDir(stage['name']) + '\n' + stage['commands'] checkResult = 'Forced' if len(onlyStages) > 0 else checkCacheKey(stage) if checkResult == 'Good': print('SKIPPING') continue elif checkResult == 'NotFound': print('NOT FOUND, ', end='') elif checkResult == 'Stale' or checkResult == 'Forced': if checkResult == 'Stale': print('CHANGED, ', end='') if rebuildStale: checkResult == 'Rebuild' else: print('(r)ebuild, rebuild (a)ll, (s)kip, (p)rint, (q)uit?: ', end='', flush=True) while True: ch = 'r' if rebuildStale else getch() if ch == 'q': finish(0) elif ch == 'p': printCommands(commands) checkResult = 'Printed' break elif ch == 's': checkResult = 'Skip' break elif ch == 'r': checkResult = 'Rebuild' break elif ch == 'a': checkResult = 'Rebuild' rebuildStale = True break if checkResult == 'Printed': continue if checkResult == 'Skip': print('SKIPPING') continue clearCacheKey(stage) print('BUILDING:') os.chdir(stage['directory']) if not run(commands): print(prefix + ': FAILED') finish(1) writeCacheKey(stage) if customRunCommand: os.chdir(executePath) command = ' '.join(runCommand) + '\n' if not run(command): print('FAILED :(') finish(1) finish(0) stage('dav1d', """ git submodule init && git submodule update cd dav1d && git reset --hard HEAD && cd .. export NDK={ndk} export NINJA_PATH=`which ninja` ./build_dav1d_clang.sh {archesStr} echo "Built archs: {archesStr}" """.format(ndk=ndkPath,archesStr=' '.join(arches))) stage('libvpx', """ git submodule init && git submodule update cd libvpx && git reset --hard HEAD && cd .. export NDK={ndk} export NINJA_PATH=`which ninja` ./build_libvpx_clang.sh {archesStr} echo "Built archs: {archesStr}" """.format(ndk=ndkPath,archesStr=' '.join(arches))) stage('ffmpeg', """ git submodule init && git submodule update cd ffmpeg && git reset --hard HEAD && cd .. export NDK={ndk} ./build_ffmpeg_clang.sh {archesStr} ./patch_ffmpeg.sh echo "Built archs: {archesStr}" """.format(ndk=ndkPath,archesStr=' '.join(arches))) stage('boringssl', """ git submodule init && git submodule update cd boringssl && git reset --hard HEAD && cd .. export NDK={ndk} export NINJA_PATH=`which ninja` ./patch_boringssl.sh ./build_boringssl.sh {archesStr} echo "Built archs: {archesStr}" """.format(ndk=ndkPath,archesStr=' '.join(arches))) stage('tde2e', """ # Install gsed on macOS. git submodule init && git submodule update git reset HEAD tde2e/ && git checkout -- tde2e/ cd tde2e_source && git reset --hard HEAD && cd .. export NDK={ndk} export NINJA_PATH=`which ninja` tde2e_dir=`pwd`/tde2e source_dir=`pwd`/tde2e_source broingssl_dir=`pwd`/boringssl cp "$tde2e_dir/build-tdlib.sh" "$source_dir/example/android/." cp "$tde2e_dir/CMakeLists.txt" "$source_dir/example/android/." for arch in {archesStr}; do mkdir -p $source_dir/example/android/third-party/openssl/$arch/lib/ cp "$broingssl_dir/build/$arch/crypto/libcrypto.a" "$source_dir/example/android/third-party/openssl/$arch/lib/libcrypto.a" cp "$broingssl_dir/build/$arch/ssl/libssl.a" "$source_dir/example/android/third-party/openssl/$arch/lib/libssl.a" cp -R "$broingssl_dir/include" "$source_dir/example/android/third-party/openssl/$arch/" done SED_CMDS="" if ! [[ "{archesStr}" =~ "arm64-v8a" ]]; then SED_CMDS+="s/arm64-v8a//g;" fi if ! [[ "{archesStr}" =~ "armeabi-v7a" ]]; then SED_CMDS+="s/armeabi-v7a//g;" fi if ! [[ "{archesStr}" =~ "x86_64" ]]; then SED_CMDS+="s/x86_64//g;" fi if ! [[ "{archesStr}" =~ "x86" ]]; then SED_CMDS+="s/x86//g;" fi comment_line() {{ SED_CMDS+="s/$1/# $1/g;" }} comment_line "rm tdlib" comment_line "jar" comment_line "mv tdlib" sed -i "s/ php//g" "$source_dir/example/android/check-environment.sh" sed -i "s/PHP_EXECUTABLE/FALSE/g" "$source_dir/td/generate/CMakeLists.txt" cd "$source_dir/example/android" if [ -n "$SED_CMDS" ]; then sed "$SED_CMDS" ./build-tdlib.sh sed "$SED_CMDS" ./build-tdlib.sh | bash -s -- "{ndk}/../.." else ./build-tdlib.sh "{ndk}/../.." fi for arch in {archesStr}; do mkdir -p $tde2e_dir/$arch cp "$source_dir/example/android/build-$arch-Java/td/tde2e/libtde2e.a" $tde2e_dir/$arch cp "$source_dir/example/android/build-$arch-Java/td/tdutils/libtdutils.a" $tde2e_dir/$arch done echo "Built archs: {archesStr}" """.format( ndk=ndkPath, archesStr=' '.join( 'arm64-v8a' if arch == 'arm64' else 'armeabi-v7a' if arch == 'arm' else arch for arch in arches ) )) runStages()