Repo created

TMessagesProj/jni/prepare.py

@@ -0,0 +1,509 @@
+import os, sys, pprint, re, json, pathlib, hashlib, subprocess, glob
+
+executePath = os.getcwd()
+scriptPath = os.path.dirname(os.path.realpath(__file__))
+
+def finish(code):
+    global executePath
+    os.chdir(executePath)
+    sys.exit(code)
+
+def error(text):
+    print('[ERROR] ' + text)
+    finish(1)
+
+arm64 = False
+arm = False
+x86 = False
+x86_64 = False
+
+ndkPath = ""
+
+win = (sys.platform == 'win32')
+mac = (sys.platform == 'darwin')
+
+dirSep = '\\' if win else '/'
+pathSep = ';' if win else ':'
+keysLoc = 'cache_keys'
+
+rootDir = os.getcwd()
+projDir = rootDir;
+print("PROJ", projDir)
+# usedPrefix = libsDir + dirSep + 'local'
+
+optionsList = [
+    'skip-release',
+    'build-qt5',
+    'skip-qt5',
+    'build-qt6',
+    'skip-qt6',
+    'build-stackwalk',
+]
+
+arches = []
+options = []
+runCommand = []
+customRunCommand = False
+for arg in sys.argv[1:]:
+    if arg == "arm64":
+        arm64 = True
+        arches.append(arg)
+        options.append(arg)
+    if arg == "arm":
+        arm = True
+        arches.append(arg)
+        options.append(arg)
+    if arg == "x86":
+        x86 = True
+        arches.append(arg)
+        options.append(arg)
+    if arg == "x86_64":
+        x86_64 = True
+        arches.append(arg)
+        options.append(arg)
+
+    if arg.startswith("ndk="):
+        ndkPath = arg.split("ndk=")[1]
+        options.append(arg)
+
+    if customRunCommand:
+        runCommand.append(arg)
+    if arg in optionsList:
+        options.append(arg)
+    elif arg == 'run':
+        customRunCommand = True
+
+if not os.path.isdir(projDir + '/' + keysLoc):
+    pathlib.Path(projDir + '/' + keysLoc).mkdir(parents=True, exist_ok=True)
+# if not os.path.isdir(libsDir + '/' + keysLoc):
+#     pathlib.Path(libsDir + '/' + keysLoc).mkdir(parents=True, exist_ok=True)
+# if not os.path.isdir(thirdPartyDir + '/' + keysLoc):
+#     pathlib.Path(thirdPartyDir + '/' + keysLoc).mkdir(parents=True, exist_ok=True)
+
+environment = {
+    'MAKE_THREADS_CNT': '-j8',
+    'MACOSX_DEPLOYMENT_TARGET': '10.12',
+    'UNGUARDED': '-Werror=unguarded-availability-new',
+    'MIN_VER': '-mmacosx-version-min=10.12',
+    'PROJ_DIR': projDir,
+}
+ignoreInCacheForThirdParty = [
+    'USED_PREFIX',
+    'LIBS_DIR',
+    'SPECIAL_TARGET',
+    'X8664',
+    'WIN32X64',
+]
+
+environmentKeyString = ''
+envForThirdPartyKeyString = ''
+for key in environment:
+    part = key + '=' + environment[key] + ';'
+    environmentKeyString += part
+    if not key in ignoreInCacheForThirdParty:
+        envForThirdPartyKeyString += part
+environmentKey = hashlib.sha1(environmentKeyString.encode('utf-8')).hexdigest()
+envForThirdPartyKey = hashlib.sha1(envForThirdPartyKeyString.encode('utf-8')).hexdigest()
+
+modifiedEnv = os.environ.copy()
+for key in environment:
+    modifiedEnv[key] = environment[key]
+
+def computeFileHash(path):
+    sha1 = hashlib.sha1()
+    with open(path, 'rb') as f:
+        while True:
+            data = f.read(256 * 1024)
+            if not data:
+                break
+            sha1.update(data)
+    return sha1.hexdigest()
+
+def computeCacheKey(stage):
+    if (stage['location'] == 'ThirdParty'):
+        envKey = envForThirdPartyKey
+    else:
+        envKey = environmentKey
+    objects = [
+        envKey,
+        stage['location'],
+        stage['name'],
+        stage['version'],
+        stage['commands']
+    ]
+    for pattern in stage['dependencies']:
+        pathlist = glob.glob(libsDir + '/' + pattern)
+        items = [pattern]
+        if len(pathlist) == 0:
+            pathlist = glob.glob(thirdPartyDir + '/' + pattern)
+        if len(pathlist) == 0:
+            error('Nothing found: ' + pattern)
+        for path in pathlist:
+            if not os.path.exists(path):
+                error('Not found: ' + path)
+            items.append(computeFileHash(path))
+        objects.append(':'.join(items))
+    return hashlib.sha1(';'.join(objects).encode('utf-8')).hexdigest()
+
+def keyPath(stage):
+    return stage['directory'] + '/' + keysLoc + '/' + stage['name']
+
+def checkCacheKey(stage):
+    if not 'key' in stage:
+        error('Key not set in stage: ' + stage['name'])
+    key = keyPath(stage)
+    if not os.path.exists(stage['directory'] + '/' + stage['name']):
+        return 'NotFound'
+    if not os.path.exists(key):
+        return 'Stale'
+    with open(key, 'r') as file:
+        return 'Good' if (file.read() == stage['key']) else 'Stale'
+
+def clearCacheKey(stage):
+    key = keyPath(stage)
+    if os.path.exists(key):
+        os.remove(key)
+
+def writeCacheKey(stage):
+    if not 'key' in stage:
+        error('Key not set in stage: ' + stage['name'])
+    key = keyPath(stage)
+    with open(key, 'w') as file:
+        file.write(stage['key'])
+
+stages = []
+
+def removeDir(folder):
+    if win:
+        return 'if exist ' + folder + ' rmdir /Q /S ' + folder + '\nif exist ' + folder + ' exit /b 1'
+    return 'rm -rf ' + folder
+
+def filterByPlatform(commands):
+    commands = commands.split('\n')
+    result = ''
+    dependencies = []
+    version = '0'
+    skip = False
+    for command in commands:
+        m = re.match(r'(!?)([a-z0-9_]+):', command)
+        if m and m.group(2) != 'depends' and m.group(2) != 'version':
+            scopes = m.group(2).split('_')
+            inscope = 'common' in scopes
+            if arm64 and 'arm64' in scopes:
+                inscope = True
+            if arm and 'arm' in scopes:
+                inscope = True
+            if x86 and 'x86' in scopes:
+                inscope = True
+            if x86_64 and 'x86_64' in scopes:
+                inscope = True
+            # if linux and 'linux' in scopes:
+            #     inscope = True
+            # if 'release' in scopes:
+            #     if 'skip-release' in options:
+            #         inscope = False
+            #     elif len(scopes) == 1:
+            #         continue
+            skip = inscope if m.group(1) == '!' else not inscope
+        elif not skip and not re.match(r'\s*#', command):
+            if m and m.group(2) == 'version':
+                version = version + '.' + command[len(m.group(0)):].strip()
+            elif m and m.group(2) == 'depends':
+                pattern = command[len(m.group(0)):].strip()
+                dependencies.append(pattern)
+            else:
+                command = command.strip()
+                if len(command) > 0:
+                    result = result + command + '\n'
+    return [result, dependencies, version]
+
+def stage(name, commands, location = '.'):
+    directory = projDir
+    # if location == 'Libraries':
+    #     directory = libsDir
+    # elif location == 'ThirdParty':
+    #     directory = thirdPartyDir
+    # else:
+    #     error('Unknown location: ' + location)
+    [commands, dependencies, version] = filterByPlatform(commands)
+    if len(commands) > 0:
+        stages.append({
+            'name': name,
+            'location': location,
+            'directory': directory,
+            'commands': commands,
+            'version': version,
+            'dependencies': dependencies
+        })
+
+def winFailOnEach(command):
+    commands = command.split('\n')
+    result = ''
+    startingCommand = True
+    for command in commands:
+        command = re.sub(r'\$([A-Za-z0-9_]+)', r'%\1%', command)
+        if re.search(r'\$', command):
+            error('Bad command: ' + command)
+        appendCall = startingCommand and not re.match(r'(if|for) ', command)
+        called = 'call ' + command if appendCall else command
+        result = result + called
+        if command.endswith('^'):
+            startingCommand = False
+        else:
+            startingCommand = True
+            result = result + '\r\nif %errorlevel% neq 0 exit /b %errorlevel%\r\n'
+    return result
+
+def printCommands(commands):
+    print('---------------------------------COMMANDS-LIST----------------------------------')
+    print(commands, end='')
+    print('--------------------------------------------------------------------------------')
+
+def run(commands):
+    printCommands(commands)
+    if win:
+        if os.path.exists("command.bat"):
+            os.remove("command.bat")
+        with open("command.bat", 'w') as file:
+            file.write('@echo OFF\r\n' + winFailOnEach(commands))
+        result = subprocess.run("command.bat", shell=True, env=modifiedEnv).returncode == 0
+        if result and os.path.exists("command.bat"):
+            os.remove("command.bat")
+        return result
+    elif re.search(r'\%', commands):
+        error('Bad command: ' + commands)
+    else:
+        return subprocess.run("set -e\n" + commands, shell=True, executable="/bin/bash", env=modifiedEnv).returncode == 0
+
+# Thanks https://stackoverflow.com/a/510364
+class _Getch:
+    """Gets a single character from standard input.  Does not echo to the
+screen."""
+    def __init__(self):
+        try:
+            self.impl = _GetchWindows()
+        except ImportError:
+            self.impl = _GetchUnix()
+
+    def __call__(self): return self.impl()
+
+class _GetchUnix:
+    def __init__(self):
+        import tty, sys
+
+    def __call__(self):
+        import sys, tty, termios
+        fd = sys.stdin.fileno()
+        old_settings = termios.tcgetattr(fd)
+        try:
+            tty.setraw(sys.stdin.fileno())
+            ch = sys.stdin.read(1)
+        finally:
+            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+        return ch
+
+class _GetchWindows:
+    def __init__(self):
+        import msvcrt
+
+    def __call__(self):
+        import msvcrt
+        return msvcrt.getch().decode('ascii')
+
+getch = _Getch()
+
+def runStages():
+    onlyStages = []
+    rebuildStale = False
+    for arg in sys.argv[1:]:
+        if arg in options:
+            continue
+        elif arg == 'silent':
+            rebuildStale = True
+            continue
+        found = False
+        for stage in stages:
+            if stage['name'] == arg:
+                onlyStages.append(arg)
+                found = True
+                break
+        if not found:
+            error('Unknown argument: ' + arg)
+    count = len(stages)
+    index = 0
+    for stage in stages:
+        if len(onlyStages) > 0 and not stage['name'] in onlyStages:
+            continue
+        index = index + 1
+        version = ('#' + str(stage['version'])) if (stage['version'] != '0') else ''
+        prefix = '[' + str(index) + '/' + str(count) + '](' + stage['location'] + '/' + stage['name'] + version + ')'
+        print(prefix + ': ', end = '', flush=True)
+        stage['key'] = computeCacheKey(stage)
+        commands = removeDir(stage['name']) + '\n' + stage['commands']
+        checkResult = 'Forced' if len(onlyStages) > 0 else checkCacheKey(stage)
+        if checkResult == 'Good':
+            print('SKIPPING')
+            continue
+        elif checkResult == 'NotFound':
+            print('NOT FOUND, ', end='')
+        elif checkResult == 'Stale' or checkResult == 'Forced':
+            if checkResult == 'Stale':
+                print('CHANGED, ', end='')
+            if rebuildStale:
+                checkResult == 'Rebuild'
+            else:
+                print('(r)ebuild, rebuild (a)ll, (s)kip, (p)rint, (q)uit?: ', end='', flush=True)
+                while True:
+                    ch = 'r' if rebuildStale else getch()
+                    if ch == 'q':
+                        finish(0)
+                    elif ch == 'p':
+                        printCommands(commands)
+                        checkResult = 'Printed'
+                        break
+                    elif ch == 's':
+                        checkResult = 'Skip'
+                        break
+                    elif ch == 'r':
+                        checkResult = 'Rebuild'
+                        break
+                    elif ch == 'a':
+                        checkResult = 'Rebuild'
+                        rebuildStale = True
+                        break
+        if checkResult == 'Printed':
+            continue
+        if checkResult == 'Skip':
+            print('SKIPPING')
+            continue
+        clearCacheKey(stage)
+        print('BUILDING:')
+        os.chdir(stage['directory'])
+        if not run(commands):
+            print(prefix + ': FAILED')
+            finish(1)
+        writeCacheKey(stage)
+
+if customRunCommand:
+    os.chdir(executePath)
+    command = ' '.join(runCommand) + '\n'
+    if not run(command):
+        print('FAILED :(')
+        finish(1)
+    finish(0)
+
+stage('dav1d', """
+    git submodule init && git submodule update
+    cd dav1d && git reset --hard HEAD && cd ..
+    export NDK={ndk}
+    export NINJA_PATH=`which ninja`
+    ./build_dav1d_clang.sh {archesStr}
+    echo "Built archs: {archesStr}"
+""".format(ndk=ndkPath,archesStr=' '.join(arches)))
+
+stage('libvpx', """
+    git submodule init && git submodule update
+    cd libvpx && git reset --hard HEAD && cd ..
+    export NDK={ndk}
+    export NINJA_PATH=`which ninja`
+    ./build_libvpx_clang.sh {archesStr}
+    echo "Built archs: {archesStr}"
+""".format(ndk=ndkPath,archesStr=' '.join(arches)))
+
+stage('ffmpeg', """
+    git submodule init && git submodule update
+    cd ffmpeg && git reset --hard HEAD && cd ..
+    export NDK={ndk}
+    ./build_ffmpeg_clang.sh {archesStr}
+    ./patch_ffmpeg.sh
+    echo "Built archs: {archesStr}"
+""".format(ndk=ndkPath,archesStr=' '.join(arches)))
+
+stage('boringssl', """
+    git submodule init && git submodule update
+    cd boringssl && git reset --hard HEAD && cd ..
+    export NDK={ndk}
+    export NINJA_PATH=`which ninja`
+    ./patch_boringssl.sh
+    ./build_boringssl.sh {archesStr}
+    echo "Built archs: {archesStr}"
+""".format(ndk=ndkPath,archesStr=' '.join(arches)))
+
+stage('tde2e', """
+    # Install gsed on macOS.
+
+    git submodule init && git submodule update
+    git reset HEAD tde2e/ && git checkout -- tde2e/
+    cd tde2e_source && git reset --hard HEAD && cd ..
+    export NDK={ndk}
+    export NINJA_PATH=`which ninja`
+
+    tde2e_dir=`pwd`/tde2e
+    source_dir=`pwd`/tde2e_source
+    broingssl_dir=`pwd`/boringssl
+
+    cp "$tde2e_dir/build-tdlib.sh" "$source_dir/example/android/."
+    cp "$tde2e_dir/CMakeLists.txt" "$source_dir/example/android/."
+
+    for arch in {archesStr}; do
+        mkdir -p $source_dir/example/android/third-party/openssl/$arch/lib/
+        cp "$broingssl_dir/build/$arch/crypto/libcrypto.a" "$source_dir/example/android/third-party/openssl/$arch/lib/libcrypto.a"
+        cp "$broingssl_dir/build/$arch/ssl/libssl.a" "$source_dir/example/android/third-party/openssl/$arch/lib/libssl.a"
+        cp -R "$broingssl_dir/include" "$source_dir/example/android/third-party/openssl/$arch/"
+    done
+
+    SED_CMDS=""
+
+    if ! [[ "{archesStr}" =~ "arm64-v8a" ]]; then
+        SED_CMDS+="s/arm64-v8a//g;"
+    fi
+
+    if ! [[ "{archesStr}" =~ "armeabi-v7a" ]]; then
+        SED_CMDS+="s/armeabi-v7a//g;"
+    fi
+
+    if ! [[ "{archesStr}" =~ "x86_64" ]]; then
+        SED_CMDS+="s/x86_64//g;"
+    fi
+
+    if ! [[ "{archesStr}" =~ "x86" ]]; then
+        SED_CMDS+="s/x86//g;"
+    fi
+
+    comment_line() {{
+        SED_CMDS+="s/$1/# $1/g;"
+    }}
+
+    comment_line "rm tdlib"
+    comment_line "jar"
+    comment_line "mv tdlib"
+
+    sed -i "s/ php//g" "$source_dir/example/android/check-environment.sh"
+    sed -i "s/PHP_EXECUTABLE/FALSE/g" "$source_dir/td/generate/CMakeLists.txt"
+
+    cd "$source_dir/example/android"
+    if [ -n "$SED_CMDS" ]; then
+        sed "$SED_CMDS" ./build-tdlib.sh
+        sed "$SED_CMDS" ./build-tdlib.sh | bash -s -- "{ndk}/../.."
+    else
+        ./build-tdlib.sh "{ndk}/../.."
+    fi
+
+    for arch in {archesStr}; do
+        mkdir -p $tde2e_dir/$arch
+        cp "$source_dir/example/android/build-$arch-Java/td/tde2e/libtde2e.a" $tde2e_dir/$arch
+        cp "$source_dir/example/android/build-$arch-Java/td/tdutils/libtdutils.a" $tde2e_dir/$arch
+    done
+
+    echo "Built archs: {archesStr}"
+""".format(
+    ndk=ndkPath,
+    archesStr=' '.join(
+        'arm64-v8a' if arch == 'arm64' else
+        'armeabi-v7a' if arch == 'arm' else
+        arch
+        for arch in arches
+    )
+))
+
+runStages()