From 8792b1691e8bd9e7b1a4d72dd25ff510a79a2a21 Mon Sep 17 00:00:00 2001 From: Xor290 Date: Sun, 10 May 2026 15:54:33 +0200 Subject: [PATCH] chore: script cache update CI client --- .../workflows/frontend-client-build-local.yml | 29 +- scripts/eas_cache.py | 268 ++++++++++++++++++ scripts/requirements.txt | 1 + 3 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 scripts/eas_cache.py create mode 100644 scripts/requirements.txt diff --git a/.github/workflows/frontend-client-build-local.yml b/.github/workflows/frontend-client-build-local.yml index b7f2235f..58aac419 100644 --- a/.github/workflows/frontend-client-build-local.yml +++ b/.github/workflows/frontend-client-build-local.yml @@ -53,8 +53,10 @@ jobs: - name: Setup Android SDK uses: android-actions/setup-android@v3 - - name: Install EAS CLI - run: npm install -g eas-cli + - name: Install EAS CLI & cache deps + run: | + npm install -g eas-cli + pip install -r scripts/requirements.txt --quiet - name: Install dependencies working-directory: mobile @@ -73,22 +75,41 @@ jobs: echo "profile=production" >> $GITHUB_OUTPUT echo "channel=production" >> $GITHUB_OUTPUT echo "api_url=https://mln-uber.club" >> $GITHUB_OUTPUT - echo "apk_name=admin-panel-production-$(date +%Y%m%d-%H%M).apk" >> $GITHUB_OUTPUT + echo "apk_name=mobile-production-$(date +%Y%m%d-%H%M).apk" >> $GITHUB_OUTPUT else echo "profile=pre-prod" >> $GITHUB_OUTPUT echo "channel=pre-prod" >> $GITHUB_OUTPUT echo "api_url=https://5.181.0.112.nip.io" >> $GITHUB_OUTPUT - echo "apk_name=admin-panel-preview-$(date +%Y%m%d-%H%M).apk" >> $GITHUB_OUTPUT + echo "apk_name=mobile-preview-$(date +%Y%m%d-%H%M).apk" >> $GITHUB_OUTPUT fi + - name: Restore Gradle cache (RustFS) + env: + AWS_ACCESS_KEY_ID: ${{ secrets.RUSTFS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + S3_ENDPOINT: https://s3.uber-stup.club + S3_BUCKET: apk-builds + run: python scripts/eas_cache.py restore --app mobile + - name: Build APK (local) working-directory: mobile env: EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }} EXPO_PUBLIC_API_URL: ${{ steps.config.outputs.api_url }} EAS_BUILD_NO_EXPO_GO_WARNING: true + GRADLE_OPTS: "-Dorg.gradle.jvmargs=-Xmx4096m -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError" + JAVA_TOOL_OPTIONS: "-Xmx4096m" run: eas build --platform android --profile ${{ steps.config.outputs.profile }} --local --non-interactive + - name: Save Gradle cache (RustFS) + if: success() || failure() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.RUSTFS_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.RUSTFS_SECRET_KEY }} + S3_ENDPOINT: https://s3.uber-stup.club + S3_BUCKET: apk-builds + run: python scripts/eas_cache.py save --app mobile + - name: Rename APK working-directory: mobile run: mv *.apk ${{ steps.config.outputs.apk_name }} diff --git a/scripts/eas_cache.py b/scripts/eas_cache.py new file mode 100644 index 00000000..733d4bc3 --- /dev/null +++ b/scripts/eas_cache.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +""" +eas_cache.py — Cache manager for EAS local builds stored on RustFS/S3 + +Reproduit le système de cache d'Expo cloud pour les builds locaux : + 1. restore : télécharge le cache Gradle depuis S3 et l'extrait dans ~/.gradle + 2. save : compresse ~/.gradle et l'upload dans S3 + +Clé de cache = sha256(package-lock.json + app.json) → premier appel exact, + sha256(package-lock.json) → fallback si app.json change + +Usage: + python scripts/eas_cache.py restore --app mobile + python scripts/eas_cache.py save --app mobile + +Variables d'environnement requises: + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + S3_ENDPOINT ex: https://s3.uber-stup.club + S3_BUCKET ex: build-cache (défaut: build-cache) +""" + +import argparse +import hashlib +import os +import sys +import tarfile +import time +from pathlib import Path + +# ── Dirs à cacher ────────────────────────────────────────────────────────────── +CACHE_DIRS = [ + Path.home() / ".gradle" / "caches" / "modules-2", # dépendances Maven/Gradle + Path.home() / ".gradle" / "caches" / "transforms-3", # artefacts transformés + Path.home() / ".gradle" / "wrapper" / "dists", # distribution Gradle +] + +# Fichiers utilisés pour calculer la clé de cache par app +HASH_FILES = { + "mobile": ["mobile/package-lock.json", "mobile/app.json"], + "frontend-admin": ["frontend-admin/package-lock.json", "frontend-admin/app.json"], +} + + +# ── Clé de cache ─────────────────────────────────────────────────────────────── + +def compute_key(app: str, repo_root: Path, full: bool = True) -> str: + """Calcule la clé de cache (sha256 tronqué à 16 chars).""" + files = HASH_FILES[app] + if not full: + files = files[:1] # fallback : seulement package-lock.json + + h = hashlib.sha256() + for rel in files: + path = repo_root / rel + if path.exists(): + h.update(path.read_bytes()) + else: + h.update(rel.encode()) # fichier absent → on hash le nom pour différencier + return h.hexdigest()[:16] + + +def s3_key(app: str, cache_key: str) -> str: + return f"gradle-cache/{app}/{cache_key}.tar.gz" + +BUCKET_DEFAULT = "apk-builds" + + +# ── Helpers S3 ───────────────────────────────────────────────────────────────── + +def _boto3_client(): + try: + import boto3 + from botocore.config import Config + except ImportError: + print("❌ boto3 non installé — lance : pip install boto3") + sys.exit(1) + + endpoint = os.environ.get("S3_ENDPOINT", "").rstrip("/") + if not endpoint: + print("❌ Variable S3_ENDPOINT manquante") + sys.exit(1) + + return boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], + region_name=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + config=Config(s3={"addressing_style": "path"}), + ) + + +class _Progress: + """Affiche la progression upload/download en MB.""" + + def __init__(self, total: int, label: str): + self.total = total + self.done = 0 + self.label = label + self.start = time.time() + + def __call__(self, chunk: int): + self.done += chunk + pct = self.done * 100 // self.total if self.total else 0 + mb_done = self.done / 1_048_576 + mb_total = self.total / 1_048_576 + elapsed = time.time() - self.start + speed = (self.done / elapsed / 1_048_576) if elapsed > 0 else 0 + print( + f"\r {self.label} {mb_done:.1f}/{mb_total:.1f} MB {pct}% {speed:.1f} MB/s", + end="", + flush=True, + ) + if self.done >= self.total: + print() + + +def object_exists(s3, bucket: str, key: str) -> bool: + try: + s3.head_object(Bucket=bucket, Key=key) + return True + except Exception: + return False + + +def object_size(s3, bucket: str, key: str) -> int: + try: + resp = s3.head_object(Bucket=bucket, Key=key) + return resp["ContentLength"] + except Exception: + return 0 + + +# ── Commande restore ─────────────────────────────────────────────────────────── + +def restore(app: str, repo_root: Path, bucket: str): + s3 = _boto3_client() + + full_key = compute_key(app, repo_root, full=True) + fallback_key = compute_key(app, repo_root, full=False) + + hit_key = None + for label, ck in [("exact", full_key), ("fallback", fallback_key)]: + obj = s3_key(app, ck) + if object_exists(s3, bucket, obj): + print(f"✅ Cache {label} trouvé → {obj}") + hit_key = ck + break + + if hit_key is None: + print("ℹ️ Aucun cache trouvé (premier build ou clé inconnue)") + return + + obj = s3_key(app, hit_key) + size = object_size(s3, bucket, obj) + progress = _Progress(size, "⬇️ download") + + import tempfile + with tempfile.TemporaryFile() as tmp: + s3.download_fileobj(bucket, obj, tmp, Callback=progress) + tmp.seek(0) + print("📦 Extraction du cache…") + with tarfile.open(fileobj=tmp, mode="r:gz") as tf: + tf.extractall(path=Path.home()) + + print(f"✅ Cache restauré ({size / 1_048_576:.1f} MB)") + + +# ── Commande save ────────────────────────────────────────────────────────────── + +def save(app: str, repo_root: Path, bucket: str): + s3 = _boto3_client() + cache_key = compute_key(app, repo_root, full=True) + obj = s3_key(app, cache_key) + + # Pas besoin de re-uploader si la clé existe déjà + if object_exists(s3, bucket, obj): + print(f"ℹ️ Cache déjà présent pour cette clé ({cache_key}), skip upload") + return + + dirs_to_cache = [d for d in CACHE_DIRS if d.exists()] + if not dirs_to_cache: + print("⚠️ Aucun répertoire Gradle à cacher (~/.gradle introuvable)") + return + + print(f"📦 Compression de {len(dirs_to_cache)} répertoires…") + for d in dirs_to_cache: + print(f" {d}") + + import io + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz", compresslevel=6) as tf: + for d in dirs_to_cache: + # Chemin relatif depuis home pour restaurer au bon endroit + arcname = str(d.relative_to(Path.home())) + tf.add(d, arcname=arcname) + + size = buf.tell() + buf.seek(0) + + print(f"⬆️ Upload {obj} ({size / 1_048_576:.1f} MB)…") + progress = _Progress(size, "⬆️ upload ") + s3.upload_fileobj(buf, bucket, obj, Callback=progress) + + print(f"✅ Cache sauvegardé ({cache_key})") + + +# ── Commande clean (optionnel) ───────────────────────────────────────────────── + +def clean(app: str, bucket: str, keep: int): + """Supprime les anciennes entrées de cache (garde les `keep` plus récentes).""" + s3 = _boto3_client() + prefix = f"gradle-cache/{app}/" + resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix) + objs = sorted( + resp.get("Contents", []), + key=lambda o: o["LastModified"], + reverse=True, + ) + to_delete = objs[keep:] + if not to_delete: + print(f"ℹ️ Rien à supprimer (≤ {keep} entrées)") + return + s3.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": o["Key"]} for o in to_delete]}, + ) + print(f"🗑 {len(to_delete)} ancienne(s) entrée(s) supprimée(s)") + + +# ── CLI ──────────────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser(description="EAS build cache manager — RustFS/S3") + parser.add_argument("command", choices=["restore", "save", "clean"]) + parser.add_argument("--app", required=True, choices=list(HASH_FILES.keys())) + parser.add_argument( + "--bucket", + default=os.environ.get("S3_BUCKET", "apk-builds"), + help="Nom du bucket S3 (défaut: apk-builds)", + ) + parser.add_argument( + "--repo-root", + default=os.environ.get("GITHUB_WORKSPACE", "."), + help="Racine du repo (défaut: GITHUB_WORKSPACE ou répertoire courant)", + ) + parser.add_argument( + "--keep", + type=int, + default=5, + help="[clean] Nombre d'entrées à conserver par app (défaut: 5)", + ) + args = parser.parse_args() + + repo_root = Path(args.repo_root).resolve() + print(f"🔧 eas_cache | app={args.app} commande={args.command} bucket={args.bucket}") + + if args.command == "restore": + restore(args.app, repo_root, args.bucket) + elif args.command == "save": + save(args.app, repo_root, args.bucket) + elif args.command == "clean": + clean(args.app, args.bucket, args.keep) + + +if __name__ == "__main__": + main() diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 00000000..011ba23b --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1 @@ +boto3>=1.34.0