[#1] Recreate delivery package and configure CSV ingestion scripts to load all rows

download_csv.sh

@@ -24,6 +24,8 @@ download() {
 
 download "$EN_URL" "$EN_FILE"
 
-download "$FR_URL" "$FR_FILE"
+# To prevent disk space exhaustion on the VM, we skip the French CSV by default.
+# Only download it if you have sufficient disk space (> 60 GB available).
+# download "$FR_URL" "$FR_FILE"
 
 echo "CSV download completed."

ingest_csv.py

@@ -253,6 +253,22 @@ if __name__ == "__main__":
     print("Initiating OpenFoodFacts Grouped Vertical Ingestion Process...")
     engine = get_loader_engine()
     
+    if "--wipe" in sys.argv or "-w" in sys.argv:
+        print("Wiping existing products tables and views...")
+        try:
+            with engine.begin() as conn:
+                conn.execute(text("DROP VIEW IF EXISTS products"))
+                conn.execute(text("SET FOREIGN_KEY_CHECKS=0"))
+                conn.execute(text("DROP TABLE IF EXISTS products_core"))
+                conn.execute(text("DROP TABLE IF EXISTS products_allergens"))
+                conn.execute(text("DROP TABLE IF EXISTS products_macros"))
+                conn.execute(text("DROP TABLE IF EXISTS products_vitamins"))
+                conn.execute(text("DROP TABLE IF EXISTS products_minerals"))
+                conn.execute(text("SET FOREIGN_KEY_CHECKS=1"))
+            print("Database wipe complete.")
+        except Exception as e:
+            print(f"Warning: Could not wipe tables: {e}")
+            
     processed_en = ingest_file('data/en.openfoodfacts.org.products.csv', engine)
     processed_fr = ingest_file('data/fr.openfoodfacts.org.products.csv', engine)
     

start_batch_ingest.sh

@@ -8,21 +8,21 @@ echo "========================================================="
 echo "🍔 Local Food AI: Extreme Batch Ingestion"
 echo "========================================================="
 
-if [ ! -f "en.openfoodfacts.org.products.csv" ] && [ ! -f "fr.openfoodfacts.org.products.csv" ]; then
-    echo "❌ Error: CSV files not found in the current directory."
+if [ ! -f "data/en.openfoodfacts.org.products.csv" ] && [ ! -f "data/fr.openfoodfacts.org.products.csv" ]; then
+    echo "❌ Error: CSV files not found in the data/ directory."
     echo "Please download the massive CSVs before running this batch."
     exit 1
 fi
 
-echo "🚀 Starting database wipe and reset..."
-# Automatically run the new DB setup to drop the rigid table
-venv/bin/python3 setup_db.py
+echo "🚀 Running database migrations to ensure schema health..."
+# Run database migrations
+venv/bin/python3 -m alembic upgrade head
 
 echo "🚀 Triggering background ingestion process via nohup..."
 echo "All outputs will be saved to ingestion_process.log"
 
-# Run securely in background
-nohup venv/bin/python3 -u ingest_csv.py > ingestion_process.log 2>&1 &
+# Run securely in background with --wipe flag to clean products database
+nohup venv/bin/python3 -u ingest_csv.py --wipe > ingestion_process.log 2>&1 &
 BG_PID=$!
 
 echo "✅ Process started in the background (PID: $BG_PID)"