chore: add script for cache
This commit is contained in:
Xor290
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
eas_cache.py — Cache manager for EAS local builds stored on RustFS/S3
|
||||
|
||||
Reproduit le système de cache d'Expo cloud pour les builds locaux :
|
||||
1. restore : télécharge le cache Gradle depuis S3 et l'extrait dans ~/.gradle
|
||||
2. save : compresse ~/.gradle et l'upload dans S3
|
||||
|
||||
Clé de cache = sha256(package-lock.json + app.json) → premier appel exact,
|
||||
sha256(package-lock.json) → fallback si app.json change
|
||||
|
||||
Usage:
|
||||
python scripts/eas_cache.py restore --app mobile
|
||||
python scripts/eas_cache.py save --app mobile
|
||||
|
||||
Variables d'environnement requises:
|
||||
AWS_ACCESS_KEY_ID
|
||||
AWS_SECRET_ACCESS_KEY
|
||||
S3_ENDPOINT ex: https://s3.uber-stup.club
|
||||
S3_BUCKET ex: build-cache (défaut: build-cache)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# ── Dirs à cacher ──────────────────────────────────────────────────────────────
|
||||
CACHE_DIRS = [
|
||||
Path.home() / ".gradle" / "caches" / "modules-2", # dépendances Maven/Gradle
|
||||
Path.home() / ".gradle" / "caches" / "transforms-3", # artefacts transformés
|
||||
Path.home() / ".gradle" / "wrapper" / "dists", # distribution Gradle
|
||||
]
|
||||
|
||||
# Fichiers utilisés pour calculer la clé de cache par app
|
||||
HASH_FILES = {
|
||||
"omnex-plateform-client": ["omnex-plateform-client/package-lock.json", "omnex-plateform-client/app.json"],
|
||||
"omnex-plateform-app": ["omnex-plateform-app/package-lock.json", "omnex-plateform-app/app.json"],
|
||||
}
|
||||
|
||||
|
||||
# ── Clé de cache ───────────────────────────────────────────────────────────────
|
||||
|
||||
def compute_key(app: str, repo_root: Path, full: bool = True) -> str:
|
||||
"""Calcule la clé de cache (sha256 tronqué à 16 chars)."""
|
||||
files = HASH_FILES[app]
|
||||
if not full:
|
||||
files = files[:1] # fallback : seulement package-lock.json
|
||||
|
||||
h = hashlib.sha256()
|
||||
for rel in files:
|
||||
path = repo_root / rel
|
||||
if path.exists():
|
||||
h.update(path.read_bytes())
|
||||
else:
|
||||
h.update(rel.encode()) # fichier absent → on hash le nom pour différencier
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def s3_key(app: str, cache_key: str) -> str:
|
||||
return f"gradle-cache/{app}/{cache_key}.tar.gz"
|
||||
|
||||
BUCKET_DEFAULT = "apk-builds"
|
||||
|
||||
|
||||
# ── Helpers S3 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _boto3_client():
|
||||
try:
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
import urllib3
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
except ImportError:
|
||||
print("❌ boto3 non installé — lance : pip install boto3")
|
||||
sys.exit(1)
|
||||
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "").rstrip("/")
|
||||
if not endpoint:
|
||||
print("❌ Variable S3_ENDPOINT manquante")
|
||||
sys.exit(1)
|
||||
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
|
||||
region_name=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"),
|
||||
config=Config(
|
||||
s3={"addressing_style": "path"},
|
||||
signature_version="s3v4",
|
||||
),
|
||||
verify=False,
|
||||
)
|
||||
|
||||
|
||||
def ensure_bucket(s3, bucket: str):
|
||||
"""Crée le bucket s'il n'existe pas."""
|
||||
try:
|
||||
s3.head_bucket(Bucket=bucket)
|
||||
except Exception:
|
||||
try:
|
||||
s3.create_bucket(Bucket=bucket)
|
||||
print(f"🪣 Bucket '{bucket}' créé")
|
||||
except Exception as e:
|
||||
print(f"❌ Impossible de créer le bucket '{bucket}': {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
class _Progress:
|
||||
"""Affiche la progression upload/download en MB."""
|
||||
|
||||
def __init__(self, total: int, label: str):
|
||||
self.total = total
|
||||
self.done = 0
|
||||
self.label = label
|
||||
self.start = time.time()
|
||||
|
||||
def __call__(self, chunk: int):
|
||||
self.done += chunk
|
||||
pct = self.done * 100 // self.total if self.total else 0
|
||||
mb_done = self.done / 1_048_576
|
||||
mb_total = self.total / 1_048_576
|
||||
elapsed = time.time() - self.start
|
||||
speed = (self.done / elapsed / 1_048_576) if elapsed > 0 else 0
|
||||
print(
|
||||
f"\r {self.label} {mb_done:.1f}/{mb_total:.1f} MB {pct}% {speed:.1f} MB/s",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
if self.done >= self.total:
|
||||
print()
|
||||
|
||||
|
||||
def object_exists(s3, bucket: str, key: str) -> bool:
|
||||
try:
|
||||
s3.head_object(Bucket=bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def object_size(s3, bucket: str, key: str) -> int:
|
||||
try:
|
||||
resp = s3.head_object(Bucket=bucket, Key=key)
|
||||
return resp["ContentLength"]
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
# ── Commande restore ───────────────────────────────────────────────────────────
|
||||
|
||||
def restore(app: str, repo_root: Path, bucket: str):
|
||||
s3 = _boto3_client()
|
||||
ensure_bucket(s3, bucket)
|
||||
|
||||
full_key = compute_key(app, repo_root, full=True)
|
||||
fallback_key = compute_key(app, repo_root, full=False)
|
||||
|
||||
hit_key = None
|
||||
for label, ck in [("exact", full_key), ("fallback", fallback_key)]:
|
||||
obj = s3_key(app, ck)
|
||||
if object_exists(s3, bucket, obj):
|
||||
print(f"✅ Cache {label} trouvé → {obj}")
|
||||
hit_key = ck
|
||||
break
|
||||
|
||||
if hit_key is None:
|
||||
print("ℹ️ Aucun cache trouvé (premier build ou clé inconnue)")
|
||||
return
|
||||
|
||||
obj = s3_key(app, hit_key)
|
||||
size = object_size(s3, bucket, obj)
|
||||
progress = _Progress(size, "⬇️ download")
|
||||
|
||||
import tempfile
|
||||
with tempfile.TemporaryFile() as tmp:
|
||||
s3.download_fileobj(bucket, obj, tmp, Callback=progress)
|
||||
tmp.seek(0)
|
||||
print("📦 Extraction du cache…")
|
||||
with tarfile.open(fileobj=tmp, mode="r:gz") as tf:
|
||||
tf.extractall(path=Path.home())
|
||||
|
||||
print(f"✅ Cache restauré ({size / 1_048_576:.1f} MB)")
|
||||
|
||||
|
||||
# ── Commande save ──────────────────────────────────────────────────────────────
|
||||
|
||||
def save(app: str, repo_root: Path, bucket: str):
|
||||
s3 = _boto3_client()
|
||||
ensure_bucket(s3, bucket)
|
||||
cache_key = compute_key(app, repo_root, full=True)
|
||||
obj = s3_key(app, cache_key)
|
||||
|
||||
# Pas besoin de re-uploader si la clé existe déjà
|
||||
if object_exists(s3, bucket, obj):
|
||||
print(f"ℹ️ Cache déjà présent pour cette clé ({cache_key}), skip upload")
|
||||
return
|
||||
|
||||
dirs_to_cache = [d for d in CACHE_DIRS if d.exists()]
|
||||
if not dirs_to_cache:
|
||||
print("⚠️ Aucun répertoire Gradle à cacher (~/.gradle introuvable)")
|
||||
return
|
||||
|
||||
print(f"📦 Compression de {len(dirs_to_cache)} répertoires…")
|
||||
for d in dirs_to_cache:
|
||||
print(f" {d}")
|
||||
|
||||
import io
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w:gz", compresslevel=6) as tf:
|
||||
for d in dirs_to_cache:
|
||||
# Chemin relatif depuis home pour restaurer au bon endroit
|
||||
arcname = str(d.relative_to(Path.home()))
|
||||
tf.add(d, arcname=arcname)
|
||||
|
||||
size = buf.tell()
|
||||
buf.seek(0)
|
||||
|
||||
print(f"⬆️ Upload {obj} ({size / 1_048_576:.1f} MB)…")
|
||||
progress = _Progress(size, "⬆️ upload ")
|
||||
s3.upload_fileobj(buf, bucket, obj, Callback=progress)
|
||||
|
||||
print(f"✅ Cache sauvegardé ({cache_key})")
|
||||
|
||||
|
||||
# ── Commande clean (optionnel) ─────────────────────────────────────────────────
|
||||
|
||||
def clean(app: str, bucket: str, keep: int):
|
||||
"""Supprime les anciennes entrées de cache (garde les `keep` plus récentes)."""
|
||||
s3 = _boto3_client()
|
||||
prefix = f"gradle-cache/{app}/"
|
||||
resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix)
|
||||
objs = sorted(
|
||||
resp.get("Contents", []),
|
||||
key=lambda o: o["LastModified"],
|
||||
reverse=True,
|
||||
)
|
||||
to_delete = objs[keep:]
|
||||
if not to_delete:
|
||||
print(f"ℹ️ Rien à supprimer (≤ {keep} entrées)")
|
||||
return
|
||||
s3.delete_objects(
|
||||
Bucket=bucket,
|
||||
Delete={"Objects": [{"Key": o["Key"]} for o in to_delete]},
|
||||
)
|
||||
print(f"🗑 {len(to_delete)} ancienne(s) entrée(s) supprimée(s)")
|
||||
|
||||
|
||||
# ── CLI ────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="EAS build cache manager — RustFS/S3")
|
||||
parser.add_argument("command", choices=["restore", "save", "clean"])
|
||||
parser.add_argument("--app", required=True, choices=list(HASH_FILES.keys()))
|
||||
parser.add_argument(
|
||||
"--bucket",
|
||||
default=os.environ.get("S3_BUCKET", "apk-builds"),
|
||||
help="Nom du bucket S3 (défaut: apk-builds)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo-root",
|
||||
default=os.environ.get("GITHUB_WORKSPACE", "."),
|
||||
help="Racine du repo (défaut: GITHUB_WORKSPACE ou répertoire courant)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep",
|
||||
type=int,
|
||||
default=5,
|
||||
help="[clean] Nombre d'entrées à conserver par app (défaut: 5)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
repo_root = Path(args.repo_root).resolve()
|
||||
print(f"🔧 eas_cache | app={args.app} commande={args.command} bucket={args.bucket}")
|
||||
|
||||
if args.command == "restore":
|
||||
restore(args.app, repo_root, args.bucket)
|
||||
elif args.command == "save":
|
||||
save(args.app, repo_root, args.bucket)
|
||||
elif args.command == "clean":
|
||||
clean(args.app, args.bucket, args.keep)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user