Add mirror fot git repo

Improve apply_patches.sh
Improve build_single.sh
Create tools to mirror all repositories in git bundles

5 files changed, 275 insertions, 0 deletions

diff --git a/tools/mirror/.gitignore b/tools/mirror/.gitignore
new file mode 100644
index 0000000..24016b3
--- /dev/null
+++ b/tools/mirror/.gitignore
@@ -0,0 +1,3 @@
+temp-list.txt
+mirror_work
+mirrors
\ No newline at end of file
diff --git a/tools/mirror/mirror.sh b/tools/mirror/mirror.sh
new file mode 100644
index 0000000..7a2ab64
--- /dev/null
+++ b/tools/mirror/mirror.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Create directories
+mkdir -p mirrors/bundles mirrors/temp
+
+# Function to convert URL to bundle name
+url_to_bundle_name() {
+    local url="$1"
+    local hash="$2"
+    # Remove https:// and .git, replace / and . with -
+    # Append first 8 chars of hash before .bundle
+    echo -n -e "$url" | sed 's|https://||' | sed 's|\.git$||' | tr '/' '-' | tr '.' '-' | tr '[:upper:]' '[:lower:]'
+    echo -n -e "-${hash:0:8}.bundle"
+}
+
+> mirrors/gitbundlemap.txt
+
+while read -r line; do
+    url=$(echo "$line" | cut -d' ' -f1)
+    commit=$(echo "$line" | cut -d' ' -f2)
+    if [ "$commit" == "b7a695cf4b41645a5f5c5a5c1f0d565b283a3585" ]; then
+        echo "Skipping $url ($commit)"
+        continue
+    fi
+    bundle_name="$(url_to_bundle_name "$url" "$commit")"
+    
+    if [ ! -f "mirrors/bundles/$bundle_name" ]; then
+
+        echo "Processing $url ($commit) -> $bundle_name"
+        
+        if curl -f -s -o "mirrors/bundles/$bundle_name" "https://static.mrcyjanek.net/download_mirror/git/$bundle_name"; then
+            echo "Downloaded existing bundle for $url ($commit)"
+        else
+            echo "No existing bundle found, cloning repository..."
+            rm -rf mirrors/temp/repo || true
+            # Clone repository
+            git clone --mirror "$url" "mirrors/temp/repo" || {
+                echo "Failed to clone $url"
+                exit 1
+            }
+            
+            # Create bundle for specific commit
+            cd mirrors/temp/repo
+            git bundle create "../../bundles/$bundle_name" --all "$commit" || {
+                echo "Failed to create bundle for $url ($commit)"
+                cd ../../..
+                exit 1
+            }
+            cd ../../..
+            
+            rm -rf mirrors/temp/repo
+        fi
+    fi
+    
+    echo "$commit $bundle_name" >> mirrors/gitbundlemap.txt
+    
+done < submodule-list.txt
+
+sort -u mirrors/gitbundlemap.txt -o mirrors/gitbundlemap.txt
+echo rsync -raz --progress $PWD/mirrors/bundles/'*' mrcyjanek@static.mrcyjanek.net:/home/mrcyjanek/web/static.mrcyjanek.net/public_html/download_mirror/git/
+echo "Done! Bundles are in mirrors/bundles/"
+echo "Commit mappings are in mirrors/gitbundlemap.txt"
\ No newline at end of file
diff --git a/tools/mirror/pack_mirror.sh b/tools/mirror/pack_mirror.sh
new file mode 100644
index 0000000..5e13708
--- /dev/null
+++ b/tools/mirror/pack_mirror.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname "$0")"
+
+MIRROR_DIR="$(pwd)/mirror_work/mirror.git"
+PACK_DIR="$(pwd)/mirror_work/pack"
+
+if [ ! -d "$MIRROR_DIR" ]; then
+    echo "Error: Mirror repository not found at $MIRROR_DIR"
+    exit 1
+fi
+
+rm -rf "$PACK_DIR"
+mkdir -p "$PACK_DIR"
+
+cp -r "$MIRROR_DIR" "$PACK_DIR/mirror.git"
+
+cd "$PACK_DIR/mirror.git"
+
+echo "Optimizing repository..."
+git gc --aggressive --prune=now
+
+git pack-refs --all
+
+rm -rf hooks/* info/* logs/* packed-refs.old
+
+git show-ref > info/refs
+
+git update-server-info
+
+mkdir -p objects
+
+while read -r hash ref; do
+    dir=${hash:0:2}
+    file=${hash:2}
+    mkdir -p "objects/$dir"
+
+    git cat-file -p "$hash" > "objects/$dir/$file"
+done < info/refs
+
+rm -rf "$TEMP_REPO"
+
+rm -f config
+
+cd "$PACK_DIR"
+
+echo rsync --delete -raz --progress "$PACK_DIR/mirror.git" "mrcyjanek@static.mrcyjanek.net:/home/mrcyjanek/web/static.mrcyjanek.net/public_html/download_mirror/git/"
+echo "Mirror packaged successfully at: $PACK_DIR/mirror.git"
\ No newline at end of file
diff --git a/tools/mirror/submodule-list.txt b/tools/mirror/submodule-list.txt
new file mode 100644
index 0000000..fe8b64e
--- /dev/null
+++ b/tools/mirror/submodule-list.txt
@@ -0,0 +1,59 @@
+https://codeberg.org/wownero/RandomWOW 27b099b6dd6fef6e17f58c6dfe00009e9c5df587
+https://codeberg.org/wownero/wownero 3e302be710f4e6b4f58642989c8e47711362fa56
+https://codeberg.org/wownero/wownero 589efde6b3d6494645c0264a0a354d039c0f46fe
+https://codeberg.org/wownero/wownero dd46a31f3cab67b316e9239b15acf7d5cea60aa9
+https://git.wownero.com/wowlet/wownero-seed.git d3f68be347facfeebbd8f68fd74982c705cb917b
+https://git.wownero.com/wownero/RandomWOW 27b099b6dd6fef6e17f58c6dfe00009e9c5df587
+https://git.wownero.com/wownero/RandomWOW 607bad48f3687c2490d90f8c55efa2dcd7cbc195
+https://git.wownero.com/wownero/wownero 1b8475003c065b0387f21323dad8a03b131ae7d1
+https://git.wownero.com/wownero/wownero 589efde6b3d6494645c0264a0a354d039c0f46fe
+https://github.com/JuliaStrings/utf8proc.git 3de4596fbe28956855df2ecb3c11c0bbc3535838
+https://github.com/MrCyjaneK/OpenSSL-for-iPhone.git 394bac11f4347cc21b4ba976c1c0629afaf67c54
+https://github.com/MrCyjaneK/OpenSSL-for-iPhone.git d3ec433a47708594d18ef6e248b66e3d64f00f0f
+https://github.com/MrCyjaneK/RandomWOW 6f30d4b924fecb231e5b683915cc75d18b3b5866
+https://github.com/MrCyjaneK/RandomX ce72c9bb9cb799e0d9171094b9abb009e04c5bfc
+https://github.com/MrCyjaneK/bc-ur d82e7c753e710b8000706dc3383b498438795208
+https://github.com/MrCyjaneK/polyseed.git bd79f5014c331273357277ed8a3d756fb61b9fa1
+https://github.com/MrCyjaneK/wownero-seed 4a4e5b6c54c1710aa185cb0838b087238cbfc9a2
+https://github.com/NLnetLabs/unbound.git cbed768b8ff9bfcf11089a5f1699b7e5707f1ea5
+https://github.com/Tencent/rapidjson 129d19ba7f496df5e33658527a7158c79b99c21c
+https://github.com/Thalhammer/jwt-cpp.git 364a5572f4b46bb9f4304cb1c92acec8ddb2c620
+https://github.com/bitcoin-core/secp256k1.git a5269373fa13ff845f654d81b90629dd78495641
+https://github.com/cake-tech/Apple-Boost-BuildScript.git 513dd145f6e67a694160b858957978e375c8d694
+https://github.com/ethereum-lists/tokens.git 88414ef852e21279f217d2f3f8fd439cd7c5fd9f
+https://github.com/google/googletest.git 0a439623f75c029912728d80cb7f1b8b48739ca4
+https://github.com/hyle-team/tor-connect.git b589edb1906dccb387cfeded6ed12286c5f0405f
+https://github.com/hyle-team/zano 2817090c8ac7639d6f697d00fc8bcba2b3681d90
+https://github.com/hyle-team/zano_ui.git 7cd0e5e54a0d692ea819b9653f60a1cd5512dc2b
+https://github.com/jedisct1/libsodium.git 3c6da4b8c27c7d546746eadabc9e2dd6c1fdfc2c
+https://github.com/jedisct1/libsodium.git 443617d7507498f7477703f0b51cb596d4539262
+https://github.com/jedisct1/libsodium.git fb80cbfa59cb97c7e867074416bfde3ec337c05a
+https://github.com/leetal/ios-cmake 06465b27698424cf4a04a5ca4904d50a3c966c45
+https://github.com/leetal/ios-cmake ad96a372b168930c2a1ff9455e1a9ccb13021617
+https://github.com/libexpat/libexpat.git 038a0bf44f940ad7f7df59cfa5ed6fcb2c63f64f
+https://github.com/libexpat/libexpat.git 3bab6c09bbe8bf42d84b81563ddbcf4cca4be838
+https://github.com/libexpat/libexpat.git 46cf63aef169f8f12ea09579db883c5251bd47d0
+https://github.com/libexpat/libexpat.git 624da0f593bb8d7e146b9f42b06d8e6c80d032a3
+https://github.com/libexpat/libexpat.git ed4090af841ebd8a7b2e367280407d74e748a7dd
+https://github.com/miniupnp/miniupnp 544e6fcc73c5ad9af48a8985c94f0f1d742ef2e0
+https://github.com/miniupnp/miniupnp b55145ec095652289a59c33603f3abafee898273
+https://github.com/miniupnp/miniupnp bd836936f7ed7e697f36dff709d00a907644172a
+https://github.com/monero-project/monero 13002ddd4b1ba4b4dcd7c47aa368a87c353b2225
+https://github.com/monero-project/monero 81d4db08eb75ce5392c65ca6571e7b08e41b7c95
+https://github.com/monero-project/monero b089f9ee69924882c5d14dd1a6991deb05d9d1cd
+https://github.com/monero-project/monero b7a695cf4b41645a5f5c5a5c1f0d565b283a3585
+https://github.com/monero-project/monero ef3e18b51beb937c7f786ecef0d0a0e3f6295082
+https://github.com/monero-project/supercop 633500ad8c8759995049ccd022107d1fa8a1bbc9
+https://github.com/tevador/RandomX 102f8acf90a7649ada410de5499a7ec62e49e1da
+https://github.com/tevador/polyseed 3ef366993258a6f1c837d4908e3e4afb2cfefff7
+https://github.com/tevador/polyseed b7c35bb3c6b91e481ecb04fc235eaff69c507fa1
+https://github.com/trezor/trezor-common.git bff7fdfe436c727982cc553bdfb29a9021b423b0
+https://github.com/wownero-mirror/wownero 589efde6b3d6494645c0264a0a354d039c0f46fe
+https://github.com/wownero-mirror/wownero dd46a31f3cab67b316e9239b15acf7d5cea60aa9
+https://github.com/wownero-mirror/wownero-seed d3f68be347facfeebbd8f68fd74982c705cb917b
+https://github.com/x2on/OpenSSL-for-iPhone.git 394bac11f4347cc21b4ba976c1c0629afaf67c54
+https://github.com/zeromq/libzmq 0ed7a08cd946e0832ac4655b7a76c09ac221f63b
+https://github.com/zeromq/libzmq 59c91227b46b60b7f965816544b25fe0a3cc8bbe
+https://github.com/zeromq/libzmq 64db7d28fea695132834f6d2c5949cfea2f22d01
+https://github.com/zeromq/libzmq 90b4f410a07222fa2e9a5f53b454a09d4533e45a
+https://github.com/zeromq/libzmq b95d94935ed107679fd0ad9efd2f3d47309b6fd3
diff --git a/tools/mirror/submodule-scanner.sh b/tools/mirror/submodule-scanner.sh
new file mode 100644
index 0000000..11b97f1
--- /dev/null
+++ b/tools/mirror/submodule-scanner.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+OUTPUT_DIR="$PWD"
+OUTPUT_FILE="${OUTPUT_DIR}/submodule-list.txt"
+TEMP_FILE="${OUTPUT_DIR}/temp-list.txt"
+
+mkdir -p "${OUTPUT_DIR}"
+
+> "${TEMP_FILE}"
+
+TEMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TEMP_DIR}"' EXIT
+
+process_current_submodules() {
+    local repo_path="$1"
+    local is_main_repo="$2"
+    local parent_path="${3:-}"
+    local original_dir="$PWD"
+    cd "${repo_path}"
+    
+    echo "DEBUG: Processing submodules in: $(pwd)"
+    
+    if [[ -f ".gitmodules" ]]; then
+        echo "DEBUG: Found .gitmodules file"
+        git config --file .gitmodules --get-regexp '^submodule\.[^\.]+\.url' | while read -r key url; do
+            name=${key#submodule.}
+            name=${name%.url}
+            
+            path=$(git config --file "${PWD}/.gitmodules" --get "submodule.${name}.path")
+            
+            if [[ -z "$path" ]]; then
+                echo "Warning: Empty path for submodule ${name}, skipping..."
+                continue
+            fi
+            
+            full_path="${parent_path:+$parent_path/}${path}"
+            
+            url=$(echo "$url" | tr -d '"' | tr -d "'")
+            
+            echo "DEBUG: Found submodule: name=${name}, path=${full_path}, url=${url}"
+            
+            if [[ -d "${path}" ]]; then
+                hash=$(git submodule status "${path}" 2>/dev/null | awk '{print $1}' | tr -d '+' | tr -d '-' || echo "")
+                if [[ "${hash}" == "b7a695cf4b41645a5f5c5a5c1f0d565b283a3585" ]]; then
+                    echo "Skipping known problematic commit: ${commit}"
+                    return 0
+                fi
+                if [[ -n "${hash}" ]]; then
+                    echo "${url} ${hash}" >> "${TEMP_FILE}"
+                    
+                    echo "DEBUG: Initializing submodule: ${path}"
+                    if git submodule update --force --init "${path}" 2>/dev/null; then
+                        if [[ -f "${path}/.gitmodules" ]]; then
+                            echo "DEBUG: Processing nested submodules in ${full_path}"
+                            process_current_submodules "${PWD}/${path}" "false" "${full_path}"
+                        fi
+                    else
+                        echo "Warning: Could not update submodule ${full_path} at commit ${hash}, skipping..."
+                        continue
+                    fi
+                else
+                    echo "Warning: Could not get hash for submodule ${full_path}, skipping..."
+                    exit 1
+                fi
+            fi
+        done
+    fi
+    cd "${original_dir}"
+}
+
+process_main_repo() {
+    local repo_path="$1"
+    cd "${repo_path}"
+
+    echo "Getting commit history for main repository..."
+    local COMMITS=$(git log --format="%H" --reverse)
+
+    for commit in $COMMITS; do
+        echo "Processing commit: ${commit}"
+        
+        git checkout --force "${commit}"
+        git reset --hard
+        
+        process_current_submodules "${repo_path}" "true"
+    done
+}
+
+echo "Cloning repository to temporary directory..."
+git clone --no-checkout https://github.com/MrCyjaneK/monero_c --branch develop "${TEMP_DIR}/main"
+
+process_main_repo "${TEMP_DIR}/main"
+
+cat "${OUTPUT_FILE}" >> "${TEMP_FILE}"
+sort -u "${TEMP_FILE}" > "${OUTPUT_FILE}"
+
+rm "${TEMP_FILE}"
+
+echo "Submodule information has been written to ${OUTPUT_FILE}"