290 lines
10 KiB
Python
290 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
eas_cache.py — Cache manager for EAS local builds stored on RustFS/S3
|
||
|
||
Reproduit le système de cache d'Expo cloud pour les builds locaux :
|
||
1. restore : télécharge le cache Gradle depuis S3 et l'extrait dans ~/.gradle
|
||
2. save : compresse ~/.gradle et l'upload dans S3
|
||
|
||
Clé de cache = sha256(package-lock.json + app.json) → premier appel exact,
|
||
sha256(package-lock.json) → fallback si app.json change
|
||
|
||
Usage:
|
||
python scripts/eas_cache.py restore --app mobile
|
||
python scripts/eas_cache.py save --app mobile
|
||
|
||
Variables d'environnement requises:
|
||
AWS_ACCESS_KEY_ID
|
||
AWS_SECRET_ACCESS_KEY
|
||
S3_ENDPOINT ex: https://s3.uber-stup.club
|
||
S3_BUCKET ex: build-cache (défaut: build-cache)
|
||
"""
|
||
|
||
import argparse
|
||
import hashlib
|
||
import os
|
||
import sys
|
||
import tarfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
# ── Dirs à cacher ──────────────────────────────────────────────────────────────
|
||
CACHE_DIRS = [
|
||
Path.home() / ".gradle" / "caches" / "modules-2", # dépendances Maven/Gradle
|
||
Path.home() / ".gradle" / "caches" / "transforms-3", # artefacts transformés
|
||
Path.home() / ".gradle" / "wrapper" / "dists", # distribution Gradle
|
||
]
|
||
|
||
# Fichiers utilisés pour calculer la clé de cache par app
|
||
HASH_FILES = {
|
||
"mobile": ["mobile/package-lock.json", "mobile/app.json"],
|
||
"frontend-admin": ["frontend-admin/package-lock.json", "frontend-admin/app.json"],
|
||
}
|
||
|
||
|
||
# ── Clé de cache ───────────────────────────────────────────────────────────────
|
||
|
||
def compute_key(app: str, repo_root: Path, full: bool = True) -> str:
|
||
"""Calcule la clé de cache (sha256 tronqué à 16 chars)."""
|
||
files = HASH_FILES[app]
|
||
if not full:
|
||
files = files[:1] # fallback : seulement package-lock.json
|
||
|
||
h = hashlib.sha256()
|
||
for rel in files:
|
||
path = repo_root / rel
|
||
if path.exists():
|
||
h.update(path.read_bytes())
|
||
else:
|
||
h.update(rel.encode()) # fichier absent → on hash le nom pour différencier
|
||
return h.hexdigest()[:16]
|
||
|
||
|
||
def s3_key(app: str, cache_key: str) -> str:
|
||
return f"gradle-cache/{app}/{cache_key}.tar.gz"
|
||
|
||
BUCKET_DEFAULT = "apk-builds"
|
||
|
||
|
||
# ── Helpers S3 ─────────────────────────────────────────────────────────────────
|
||
|
||
def _boto3_client():
|
||
try:
|
||
import boto3
|
||
from botocore.config import Config
|
||
import urllib3
|
||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||
except ImportError:
|
||
print("❌ boto3 non installé — lance : pip install boto3")
|
||
sys.exit(1)
|
||
|
||
endpoint = os.environ.get("S3_ENDPOINT", "").rstrip("/")
|
||
if not endpoint:
|
||
print("❌ Variable S3_ENDPOINT manquante")
|
||
sys.exit(1)
|
||
|
||
return boto3.client(
|
||
"s3",
|
||
endpoint_url=endpoint,
|
||
aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
|
||
aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
|
||
region_name=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"),
|
||
config=Config(
|
||
s3={"addressing_style": "path"},
|
||
signature_version="s3v4",
|
||
),
|
||
verify=False,
|
||
)
|
||
|
||
|
||
def ensure_bucket(s3, bucket: str):
|
||
"""Crée le bucket s'il n'existe pas."""
|
||
try:
|
||
s3.head_bucket(Bucket=bucket)
|
||
except Exception:
|
||
try:
|
||
s3.create_bucket(Bucket=bucket)
|
||
print(f"🪣 Bucket '{bucket}' créé")
|
||
except Exception as e:
|
||
print(f"❌ Impossible de créer le bucket '{bucket}': {e}")
|
||
sys.exit(1)
|
||
|
||
|
||
class _Progress:
|
||
"""Affiche la progression upload/download en MB."""
|
||
|
||
def __init__(self, total: int, label: str):
|
||
self.total = total
|
||
self.done = 0
|
||
self.label = label
|
||
self.start = time.time()
|
||
|
||
def __call__(self, chunk: int):
|
||
self.done += chunk
|
||
pct = self.done * 100 // self.total if self.total else 0
|
||
mb_done = self.done / 1_048_576
|
||
mb_total = self.total / 1_048_576
|
||
elapsed = time.time() - self.start
|
||
speed = (self.done / elapsed / 1_048_576) if elapsed > 0 else 0
|
||
print(
|
||
f"\r {self.label} {mb_done:.1f}/{mb_total:.1f} MB {pct}% {speed:.1f} MB/s",
|
||
end="",
|
||
flush=True,
|
||
)
|
||
if self.done >= self.total:
|
||
print()
|
||
|
||
|
||
def object_exists(s3, bucket: str, key: str) -> bool:
|
||
try:
|
||
s3.head_object(Bucket=bucket, Key=key)
|
||
return True
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def object_size(s3, bucket: str, key: str) -> int:
|
||
try:
|
||
resp = s3.head_object(Bucket=bucket, Key=key)
|
||
return resp["ContentLength"]
|
||
except Exception:
|
||
return 0
|
||
|
||
|
||
# ── Commande restore ───────────────────────────────────────────────────────────
|
||
|
||
def restore(app: str, repo_root: Path, bucket: str):
|
||
s3 = _boto3_client()
|
||
ensure_bucket(s3, bucket)
|
||
|
||
full_key = compute_key(app, repo_root, full=True)
|
||
fallback_key = compute_key(app, repo_root, full=False)
|
||
|
||
hit_key = None
|
||
for label, ck in [("exact", full_key), ("fallback", fallback_key)]:
|
||
obj = s3_key(app, ck)
|
||
if object_exists(s3, bucket, obj):
|
||
print(f"✅ Cache {label} trouvé → {obj}")
|
||
hit_key = ck
|
||
break
|
||
|
||
if hit_key is None:
|
||
print("ℹ️ Aucun cache trouvé (premier build ou clé inconnue)")
|
||
return
|
||
|
||
obj = s3_key(app, hit_key)
|
||
size = object_size(s3, bucket, obj)
|
||
progress = _Progress(size, "⬇️ download")
|
||
|
||
import tempfile
|
||
with tempfile.TemporaryFile() as tmp:
|
||
s3.download_fileobj(bucket, obj, tmp, Callback=progress)
|
||
tmp.seek(0)
|
||
print("📦 Extraction du cache…")
|
||
with tarfile.open(fileobj=tmp, mode="r:gz") as tf:
|
||
tf.extractall(path=Path.home())
|
||
|
||
print(f"✅ Cache restauré ({size / 1_048_576:.1f} MB)")
|
||
|
||
|
||
# ── Commande save ──────────────────────────────────────────────────────────────
|
||
|
||
def save(app: str, repo_root: Path, bucket: str):
|
||
s3 = _boto3_client()
|
||
ensure_bucket(s3, bucket)
|
||
cache_key = compute_key(app, repo_root, full=True)
|
||
obj = s3_key(app, cache_key)
|
||
|
||
# Pas besoin de re-uploader si la clé existe déjà
|
||
if object_exists(s3, bucket, obj):
|
||
print(f"ℹ️ Cache déjà présent pour cette clé ({cache_key}), skip upload")
|
||
return
|
||
|
||
dirs_to_cache = [d for d in CACHE_DIRS if d.exists()]
|
||
if not dirs_to_cache:
|
||
print("⚠️ Aucun répertoire Gradle à cacher (~/.gradle introuvable)")
|
||
return
|
||
|
||
print(f"📦 Compression de {len(dirs_to_cache)} répertoires…")
|
||
for d in dirs_to_cache:
|
||
print(f" {d}")
|
||
|
||
import io
|
||
buf = io.BytesIO()
|
||
with tarfile.open(fileobj=buf, mode="w:gz", compresslevel=6) as tf:
|
||
for d in dirs_to_cache:
|
||
# Chemin relatif depuis home pour restaurer au bon endroit
|
||
arcname = str(d.relative_to(Path.home()))
|
||
tf.add(d, arcname=arcname)
|
||
|
||
size = buf.tell()
|
||
buf.seek(0)
|
||
|
||
print(f"⬆️ Upload {obj} ({size / 1_048_576:.1f} MB)…")
|
||
progress = _Progress(size, "⬆️ upload ")
|
||
s3.upload_fileobj(buf, bucket, obj, Callback=progress)
|
||
|
||
print(f"✅ Cache sauvegardé ({cache_key})")
|
||
|
||
|
||
# ── Commande clean (optionnel) ─────────────────────────────────────────────────
|
||
|
||
def clean(app: str, bucket: str, keep: int):
|
||
"""Supprime les anciennes entrées de cache (garde les `keep` plus récentes)."""
|
||
s3 = _boto3_client()
|
||
prefix = f"gradle-cache/{app}/"
|
||
resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix)
|
||
objs = sorted(
|
||
resp.get("Contents", []),
|
||
key=lambda o: o["LastModified"],
|
||
reverse=True,
|
||
)
|
||
to_delete = objs[keep:]
|
||
if not to_delete:
|
||
print(f"ℹ️ Rien à supprimer (≤ {keep} entrées)")
|
||
return
|
||
s3.delete_objects(
|
||
Bucket=bucket,
|
||
Delete={"Objects": [{"Key": o["Key"]} for o in to_delete]},
|
||
)
|
||
print(f"🗑 {len(to_delete)} ancienne(s) entrée(s) supprimée(s)")
|
||
|
||
|
||
# ── CLI ────────────────────────────────────────────────────────────────────────
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="EAS build cache manager — RustFS/S3")
|
||
parser.add_argument("command", choices=["restore", "save", "clean"])
|
||
parser.add_argument("--app", required=True, choices=list(HASH_FILES.keys()))
|
||
parser.add_argument(
|
||
"--bucket",
|
||
default=os.environ.get("S3_BUCKET", "apk-builds"),
|
||
help="Nom du bucket S3 (défaut: apk-builds)",
|
||
)
|
||
parser.add_argument(
|
||
"--repo-root",
|
||
default=os.environ.get("GITHUB_WORKSPACE", "."),
|
||
help="Racine du repo (défaut: GITHUB_WORKSPACE ou répertoire courant)",
|
||
)
|
||
parser.add_argument(
|
||
"--keep",
|
||
type=int,
|
||
default=5,
|
||
help="[clean] Nombre d'entrées à conserver par app (défaut: 5)",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
repo_root = Path(args.repo_root).resolve()
|
||
print(f"🔧 eas_cache | app={args.app} commande={args.command} bucket={args.bucket}")
|
||
|
||
if args.command == "restore":
|
||
restore(args.app, repo_root, args.bucket)
|
||
elif args.command == "save":
|
||
save(args.app, repo_root, args.bucket)
|
||
elif args.command == "clean":
|
||
clean(args.app, args.bucket, args.keep)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|