chore: add scripts
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import adresse_correction depuis data2.txt (format psql pipe-séparé, avec
|
||||
gestion des valeurs multi-lignes et des lignes fusionnées sans newline).
|
||||
Usage : python3 import_adresse_correction.py [--dry-run]
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
DB_CONFIG = {
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"user": "",
|
||||
"password": "",
|
||||
"dbname": "",
|
||||
}
|
||||
|
||||
DATA_FILE = os.path.join(os.path.dirname(__file__), "data2.txt")
|
||||
DRY_RUN = "--dry-run" in sys.argv
|
||||
|
||||
SOURCE_COLUMNS = [
|
||||
"id",
|
||||
"invalid_address",
|
||||
"correct_address",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
|
||||
|
||||
def preprocess(content: str) -> list[str]:
|
||||
"""
|
||||
Sépare les lignes physiquement fusionnées :
|
||||
ex: "...2026-03-13 22:33:10.353601 79 | 55 Quai..." → deux lignes.
|
||||
Pattern : timestamp immédiatement suivi (sans \n) d'un entier + '|'.
|
||||
"""
|
||||
content = re.sub(
|
||||
r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)\s{2,}(\d+\s*\|)",
|
||||
r"\1\n \2",
|
||||
content,
|
||||
)
|
||||
return content.splitlines()
|
||||
|
||||
|
||||
def parse_psql_rows(path: str) -> list[list[str]]:
|
||||
"""
|
||||
Parse le format aligné psql :
|
||||
- Lignes de continuation (sans id, commençant par ' |') : valeur multi-ligne.
|
||||
- Champ se terminant par '+' avant '|' : continuation sur la ligne suivante.
|
||||
"""
|
||||
with open(path, encoding="utf-8") as f:
|
||||
raw = f.read()
|
||||
|
||||
lines = preprocess(raw)
|
||||
rows: list[list[str]] = []
|
||||
current: list[str] | None = None
|
||||
|
||||
for line in lines:
|
||||
line = line.rstrip("\n")
|
||||
stripped = line.strip()
|
||||
|
||||
# Ligne vide
|
||||
if not stripped:
|
||||
continue
|
||||
|
||||
# Ligne séparateur psql (----+----)
|
||||
if (
|
||||
set(
|
||||
stripped.replace("|", "")
|
||||
.replace("+", "")
|
||||
.replace("-", "")
|
||||
.replace(" ", "")
|
||||
)
|
||||
== set()
|
||||
):
|
||||
continue
|
||||
|
||||
parts = line.split("|")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
|
||||
first = parts[0].strip()
|
||||
is_continuation = current is not None and (first == "" or not first.isdigit())
|
||||
|
||||
if is_continuation:
|
||||
# Fusionner avec la ligne en cours
|
||||
cont = parts[:-1] if parts[-1].strip() == "" else parts
|
||||
for i, cf in enumerate(cont):
|
||||
if i < len(current):
|
||||
prev = current[i].rstrip("+").rstrip()
|
||||
add = cf.strip()
|
||||
current[i] = (prev + (" " if prev and add else "") + add).strip()
|
||||
continue
|
||||
|
||||
# Sauvegarder la ligne précédente
|
||||
if current is not None:
|
||||
rows.append(current)
|
||||
|
||||
# Nouvelle ligne de données : nettoyer le '+' de continuation
|
||||
raw_fields = parts[:-1] if parts[-1].strip() == "" else parts
|
||||
current = [f.strip().rstrip("+").strip() for f in raw_fields]
|
||||
|
||||
if current is not None:
|
||||
rows.append(current)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def parse_value(v: str):
|
||||
v = v.strip()
|
||||
if v == "" or v.lower() == "null":
|
||||
return None
|
||||
return v
|
||||
|
||||
|
||||
def main():
|
||||
raw_rows = parse_psql_rows(DATA_FILE)
|
||||
if not raw_rows:
|
||||
print("❌ Aucune ligne trouvée dans data2.txt")
|
||||
sys.exit(1)
|
||||
|
||||
# Filtrer l'en-tête éventuel
|
||||
rows = [[parse_value(v) for v in r] for r in raw_rows if r[0].strip() != "id"]
|
||||
|
||||
print(f"📂 {len(rows)} lignes de données à importer depuis {DATA_FILE}")
|
||||
|
||||
insert_cols = [c for c in SOURCE_COLUMNS if c != "id"]
|
||||
insert_cols_all = ["id"] + insert_cols
|
||||
placeholders = ", ".join(["%s"] * len(insert_cols_all))
|
||||
col_names = ", ".join(insert_cols_all)
|
||||
update_expr = ", ".join(f"{c} = EXCLUDED.{c}" for c in insert_cols)
|
||||
|
||||
sql = f"""
|
||||
INSERT INTO adresse_correction ({col_names})
|
||||
VALUES ({placeholders})
|
||||
ON CONFLICT (invalid_address) DO UPDATE SET {update_expr}
|
||||
"""
|
||||
|
||||
conn = psycopg2.connect(**DB_CONFIG)
|
||||
conn.autocommit = False
|
||||
cur = conn.cursor()
|
||||
|
||||
inserted = updated = errors = 0
|
||||
|
||||
for i, row in enumerate(rows, 1):
|
||||
if len(row) < len(SOURCE_COLUMNS):
|
||||
print(f" ⚠️ Ligne {i} ignorée ({len(row)} champs) : {row}")
|
||||
continue
|
||||
|
||||
row_dict = {SOURCE_COLUMNS[j]: row[j] for j in range(len(SOURCE_COLUMNS))}
|
||||
values = [row_dict[c] for c in insert_cols_all]
|
||||
inv_addr = row_dict.get("invalid_address", "?")
|
||||
|
||||
if DRY_RUN:
|
||||
print(
|
||||
f" [DRY-RUN] {i:3}. {inv_addr!r} → {row_dict.get('correct_address')!r}"
|
||||
)
|
||||
inserted += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM adresse_correction WHERE invalid_address = %s",
|
||||
(inv_addr,),
|
||||
)
|
||||
exists = cur.fetchone() is not None
|
||||
cur.execute(sql, values)
|
||||
if exists:
|
||||
updated += 1
|
||||
else:
|
||||
inserted += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Erreur ligne {i} ({inv_addr!r}) : {e}")
|
||||
conn.rollback()
|
||||
errors += 1
|
||||
|
||||
if not DRY_RUN:
|
||||
cur.execute(
|
||||
"SELECT setval('adresse_correction_id_seq', COALESCE((SELECT MAX(id) FROM adresse_correction), 1))"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
print(f"\n{'🔍 DRY-RUN — ' if DRY_RUN else ''}✅ Terminé")
|
||||
print(f" Insérés : {inserted}")
|
||||
print(f" Mis à jour : {updated}")
|
||||
print(f" Erreurs : {errors}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import clients depuis data.txt (format psql pipe-séparé) vers PostgreSQL.
|
||||
Usage : python3 import_clients.py [--dry-run]
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
# ── Connexion ──────────────────────────────────────────────────────────────────
|
||||
DB_CONFIG = {
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"user": "",
|
||||
"password": "",
|
||||
"dbname": "",
|
||||
}
|
||||
|
||||
DATA_FILE = os.path.join(os.path.dirname(__file__), "data.txt")
|
||||
DRY_RUN = "--dry-run" in sys.argv
|
||||
|
||||
# ── Ordre des colonnes dans data.txt (DB production, construite incrémentalement)
|
||||
# None = colonne inconnue/obsolète à ignorer
|
||||
SOURCE_COLUMNS = [
|
||||
"id",
|
||||
"username",
|
||||
"password",
|
||||
"nom",
|
||||
"prenom",
|
||||
"telephone",
|
||||
"command",
|
||||
"amende",
|
||||
"cancel_commande",
|
||||
"referral_balance", # position 9 : NUMERIC(10,2) → "0.00"
|
||||
"cancellations_count", # position 10 : INTEGER → "0"
|
||||
None, # position 11 : colonne obsolète/inconnue, ignorée
|
||||
"last_penalty_reason", # position 12 : TEXT
|
||||
"updated_at",
|
||||
"created_at",
|
||||
"must_change_password", # position 15 : BOOLEAN → "f"/"t"
|
||||
]
|
||||
|
||||
|
||||
def parse_value(v: str):
|
||||
"""Convertit une valeur brute psql en type Python (None si vide)."""
|
||||
v = v.strip()
|
||||
if v == "" or v.lower() == "null":
|
||||
return None
|
||||
if v.lower() == "t":
|
||||
return True
|
||||
if v.lower() == "f":
|
||||
return False
|
||||
return v
|
||||
|
||||
|
||||
def load_data(path: str) -> list[list]:
|
||||
rows = []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.rstrip("\n")
|
||||
if not line.strip():
|
||||
continue
|
||||
# Lignes séparateur psql (ex: "----+----")
|
||||
if (
|
||||
set(
|
||||
line.replace("|", "")
|
||||
.replace("+", "")
|
||||
.replace("-", "")
|
||||
.replace(" ", "")
|
||||
)
|
||||
== set()
|
||||
):
|
||||
continue
|
||||
parts = line.split("|")
|
||||
values = (
|
||||
[parse_value(p) for p in parts[:-1]]
|
||||
if parts[-1].strip() == ""
|
||||
else [parse_value(p) for p in parts]
|
||||
)
|
||||
if values:
|
||||
rows.append(values)
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
rows = load_data(DATA_FILE)
|
||||
if not rows:
|
||||
print("❌ Aucune ligne trouvée dans data.txt")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"📂 {len(rows)} lignes lues depuis {DATA_FILE}")
|
||||
|
||||
data_col_count = len(rows[0])
|
||||
print(
|
||||
f"📋 Colonnes data.txt : {data_col_count} | SOURCE_COLUMNS définis : {len(SOURCE_COLUMNS)}"
|
||||
)
|
||||
|
||||
if data_col_count != len(SOURCE_COLUMNS):
|
||||
print(
|
||||
f"⚠️ Ajuste SOURCE_COLUMNS dans le script ({len(SOURCE_COLUMNS)} définis, {data_col_count} dans le fichier)"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# Colonnes à insérer (on filtre les None = colonnes ignorées)
|
||||
insert_cols = [col for col in SOURCE_COLUMNS if col is not None and col != "id"]
|
||||
# On conserve aussi l'id pour FORCE_ID
|
||||
insert_cols_with_id = ["id"] + insert_cols
|
||||
|
||||
placeholders = ", ".join(["%s"] * len(insert_cols_with_id))
|
||||
col_names = ", ".join(insert_cols_with_id)
|
||||
update_expr = ", ".join(f"{c} = EXCLUDED.{c}" for c in insert_cols)
|
||||
|
||||
sql = f"""
|
||||
INSERT INTO clients ({col_names})
|
||||
VALUES ({placeholders})
|
||||
ON CONFLICT (username) DO UPDATE SET {update_expr}
|
||||
"""
|
||||
|
||||
conn = psycopg2.connect(**DB_CONFIG)
|
||||
conn.autocommit = False
|
||||
cur = conn.cursor()
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
errors = 0
|
||||
|
||||
for i, row in enumerate(rows, 1):
|
||||
# Construire un dict {col_name: value} depuis SOURCE_COLUMNS
|
||||
row_dict = {}
|
||||
for pos, col in enumerate(SOURCE_COLUMNS):
|
||||
if col is None:
|
||||
continue
|
||||
row_dict[col] = row[pos] if pos < len(row) else None
|
||||
|
||||
values = [row_dict.get(c) for c in insert_cols_with_id]
|
||||
username = row_dict.get("username", "?")
|
||||
|
||||
try:
|
||||
if DRY_RUN:
|
||||
print(f" [DRY-RUN] ligne {i} : {username} → {row_dict}")
|
||||
inserted += 1
|
||||
continue
|
||||
|
||||
cur.execute("SELECT 1 FROM clients WHERE username = %s", (username,))
|
||||
exists = cur.fetchone() is not None
|
||||
|
||||
cur.execute(sql, values)
|
||||
|
||||
if exists:
|
||||
updated += 1
|
||||
else:
|
||||
inserted += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Erreur ligne {i} ({username}) : {e}")
|
||||
conn.rollback()
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
if not DRY_RUN:
|
||||
cur.execute(
|
||||
"SELECT setval('clients_id_seq', COALESCE((SELECT MAX(id) FROM clients), 1))"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
print(f"\n{'🔍 DRY-RUN — ' if DRY_RUN else ''}✅ Terminé")
|
||||
print(f" Insérés : {inserted}")
|
||||
print(f" Mis à jour : {updated}")
|
||||
print(f" Erreurs : {errors}")
|
||||
print(f" ℹ️ Séquence clients_id_seq resynchronisée sur MAX(id)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import users depuis data3.txt (format psql pipe-séparé) vers PostgreSQL.
|
||||
Usage : python3 import_users.py [--dry-run]
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
DB_CONFIG = {
|
||||
"host": "localhost",
|
||||
"port": 5432,
|
||||
"user": "",
|
||||
"password": "",
|
||||
"dbname": "",
|
||||
}
|
||||
|
||||
DATA_FILE = os.path.join(os.path.dirname(__file__), "data3.txt")
|
||||
DRY_RUN = "--dry-run" in sys.argv
|
||||
|
||||
# Colonnes dans data3.txt — None = ignoré (push_token supprimé)
|
||||
SOURCE_COLUMNS = [
|
||||
"id",
|
||||
"username",
|
||||
"password",
|
||||
"role",
|
||||
"total",
|
||||
"livraison",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
|
||||
|
||||
def parse_value(v: str):
|
||||
v = v.strip()
|
||||
if v == "" or v.lower() == "null":
|
||||
return None
|
||||
if v.lower() == "t":
|
||||
return True
|
||||
if v.lower() == "f":
|
||||
return False
|
||||
return v
|
||||
|
||||
|
||||
def load_data(path: str) -> list[list]:
|
||||
rows = []
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.rstrip("\n")
|
||||
if not line.strip():
|
||||
continue
|
||||
if (
|
||||
set(
|
||||
line.replace("|", "")
|
||||
.replace("+", "")
|
||||
.replace("-", "")
|
||||
.replace(" ", "")
|
||||
)
|
||||
== set()
|
||||
):
|
||||
continue
|
||||
parts = line.split("|")
|
||||
values = (
|
||||
[parse_value(p) for p in parts[:-1]]
|
||||
if parts[-1].strip() == ""
|
||||
else [parse_value(p) for p in parts]
|
||||
)
|
||||
# Ignorer la ligne d'en-tête
|
||||
if values and values[0] == "id":
|
||||
continue
|
||||
if values:
|
||||
rows.append(values)
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
rows = load_data(DATA_FILE)
|
||||
if not rows:
|
||||
print("❌ Aucune ligne trouvée dans data3.txt")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"📂 {len(rows)} lignes lues depuis {DATA_FILE}")
|
||||
|
||||
data_col_count = len(rows[0])
|
||||
print(
|
||||
f"📋 Colonnes data3.txt : {data_col_count} | SOURCE_COLUMNS définis : {len(SOURCE_COLUMNS)}"
|
||||
)
|
||||
|
||||
if data_col_count != len(SOURCE_COLUMNS):
|
||||
print(
|
||||
f"⚠️ Ajuste SOURCE_COLUMNS ({len(SOURCE_COLUMNS)} définis, {data_col_count} dans le fichier)"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
insert_cols = [col for col in SOURCE_COLUMNS if col is not None and col != "id"]
|
||||
insert_cols_with_id = ["id"] + insert_cols
|
||||
|
||||
placeholders = ", ".join(["%s"] * len(insert_cols_with_id))
|
||||
col_names = ", ".join(insert_cols_with_id)
|
||||
update_expr = ", ".join(f"{c} = EXCLUDED.{c}" for c in insert_cols)
|
||||
|
||||
sql = f"""
|
||||
INSERT INTO users ({col_names})
|
||||
VALUES ({placeholders})
|
||||
ON CONFLICT (username) DO UPDATE SET {update_expr}
|
||||
"""
|
||||
|
||||
conn = psycopg2.connect(**DB_CONFIG)
|
||||
conn.autocommit = False
|
||||
cur = conn.cursor()
|
||||
|
||||
inserted = updated = errors = 0
|
||||
|
||||
for i, row in enumerate(rows, 1):
|
||||
row_dict = {}
|
||||
for pos, col in enumerate(SOURCE_COLUMNS):
|
||||
if col is None:
|
||||
continue
|
||||
row_dict[col] = row[pos] if pos < len(row) else None
|
||||
|
||||
values = [row_dict.get(c) for c in insert_cols_with_id]
|
||||
username = row_dict.get("username", "?")
|
||||
|
||||
try:
|
||||
if DRY_RUN:
|
||||
print(f" [DRY-RUN] {i}. {username} → {row_dict}")
|
||||
inserted += 1
|
||||
continue
|
||||
|
||||
cur.execute("SELECT 1 FROM users WHERE username = %s", (username,))
|
||||
exists = cur.fetchone() is not None
|
||||
cur.execute(sql, values)
|
||||
if exists:
|
||||
updated += 1
|
||||
else:
|
||||
inserted += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Erreur ligne {i} ({username}) : {e}")
|
||||
conn.rollback()
|
||||
errors += 1
|
||||
|
||||
if not DRY_RUN:
|
||||
cur.execute(
|
||||
"SELECT setval('users_id_seq', COALESCE((SELECT MAX(id) FROM users), 1))"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
print(f"\n{'🔍 DRY-RUN — ' if DRY_RUN else ''}✅ Terminé")
|
||||
print(f" Insérés : {inserted}")
|
||||
print(f" Mis à jour : {updated}")
|
||||
print(f" Erreurs : {errors}")
|
||||
print(" ℹ️ Séquence users_id_seq resynchronisée sur MAX(id)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user