LocalFoodAI_lanfr144/start_batch_ingest.sh

#!/bin/bash
#ident "@(#)$Format:LocalFoodAI_lanfr144:start_batch_ingest.sh:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
# Local Food AI - Disconnected Ingestion Wrapper
# This script uses nohup to run the python ingestion script in the background.
# You can exit your SSH session safely after starting this script.

echo "========================================================="
echo "🍔 Local Food AI: Extreme Batch Ingestion"
echo "========================================================="

if [ ! -f "data/en.openfoodfacts.org.products.csv" ] && [ ! -f "data/fr.openfoodfacts.org.products.csv" ]; then
    echo "❌ Error: CSV files not found in the data/ directory."
    echo "Please download the massive CSVs before running this batch."
    exit 1
fi

# Load DB environment variables for ingestion and migrations from .env
if [ -f .env ]; then
    DB_LOADER_PASS=$(grep '^DB_LOADER_PASS=' .env | cut -d'=' -f2- | tr -d '\r')
    export DB_HOST="127.0.0.1"
    export DB_USER="food_loader"
    export DB_PASS="$DB_LOADER_PASS"
fi

echo "🚀 Running database migrations to ensure schema health..."
# Run database migrations
venv/bin/python3 -m alembic upgrade head

echo "🚀 Triggering background ingestion process via nohup..."
echo "All outputs will be saved to ingestion_process.log"

# Run securely in background with --wipe flag to clean products database
nohup venv/bin/python3 -u ingest_csv.py --wipe > ingestion_process.log 2>&1 &
BG_PID=$!

echo "✅ Process started in the background (PID: $BG_PID)"
echo "You can now safely close your terminal or turn off your computer."
echo "To monitor progress from the server later, run:"
echo "   tail -f ingestion_process.log"