generate_docs.py 50 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148
  1. #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  2. import os
  3. import subprocess
  4. import re
  5. docs_dir = "docs"
  6. os.makedirs(docs_dir, exist_ok=True)
  7. docs = {
  8. "URL_Formats.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  9. # Local Food AI - Network Connection URL Directory
  10. This runbook catalogs the specific network formats and port endpoints required to access the application and monitoring servers across different loopback, hostname, and address protocols.
  11. ## 1. Localhost Format (Loopback)
  12. - **Streamlit Web Application UI**: `http://localhost:100` *(via Nginx)* or `http://localhost:8522` *(direct)*
  13. - **Zabbix Web UI Console**: `http://localhost:8101`
  14. - **Airflow Webserver DAG UI**: `http://localhost:8102`
  15. - **Ollama AI Local Engine**: `http://localhost:11434`
  16. - **SearXNG Meta-Search API**: `http://localhost:8105`
  17. - **MySQL Database Server**: `localhost:3326` *(direct SQL connection)*
  18. ## 2. Hostname Format (assuming Hostname is `XYZZYX`)
  19. - **Streamlit Web Application UI**: `http://XYZZYX:100` or `http://XYZZYX:8522`
  20. - **Zabbix Web UI Console**: `http://XYZZYX:8101`
  21. - **Airflow Webserver DAG UI**: `http://XYZZYX:8102`
  22. - **Ollama AI Local Engine**: `http://XYZZYX:11434`
  23. - **SearXNG Meta-Search API**: `http://XYZZYX:8105`
  24. - **MySQL Database Server**: `XYZZYX:3326`
  25. ## 3. IPv4 Format (assuming Local Host IP is `192.168.1.50`)
  26. - **Streamlit Web Application UI**: `http://192.168.1.50:100` or `http://192.168.1.50:8522` *(loopback: `http://127.0.0.1:100`)*
  27. - **Zabbix Web UI Console**: `http://192.168.1.50:8101` *(loopback: `http://127.0.0.1:8101`)*
  28. - **Airflow Webserver DAG UI**: `http://192.168.1.50:8102` *(loopback: `http://127.0.0.1:8102`)*
  29. - **Ollama AI Local Engine**: `http://192.168.1.50:11434` *(loopback: `http://127.0.0.1:11434`)*
  30. - **SearXNG Meta-Search API**: `http://192.168.1.50:8105` *(loopback: `http://127.0.0.1:8105`)*
  31. - **MySQL Database Server**: `192.168.1.50:3326` *(loopback: `127.0.0.1:3326`)*
  32. ## 4. IPv6 Format (using loopback `[::1]` or link-local address)
  33. - **Streamlit Web Application UI**: `http://[::1]:100` or `http://[::1]:8522`
  34. - **Zabbix Web UI Console**: `http://[::1]:8101`
  35. - **Airflow Webserver DAG UI**: `http://[::1]:8102`
  36. - **Ollama AI Local Engine**: `http://[::1]:11434`
  37. - **SearXNG Meta-Search API**: `http://[::1]:8105`
  38. - **MySQL Database Server**: `[::1]:3326`
  39. """,
  40. "Final_Report.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  41. # Final Project Report (Living Document)
  42. ## What Has Been Done
  43. 1. **Core Architecture**: Deployed a resilient 8-container local fallback Docker Compose stack (MySQL, Streamlit UI, local Ollama LLM, anonymous SearXNG search, secure Nginx proxy, and local Zabbix Server/Web/Agent observability suite).
  44. 2. **Database Optimization**: Successfully loaded OpenFoodFacts records and utilized advanced vertical partitioning and FULLTEXT indices.
  45. 3. **Clinical Subquery Strategy**: Refactored the core Pandas/SQL query pipeline to use subquery limiting, resolving Cartesian join explosions and reducing query latency to ~0.04s.
  46. 4. **Monitoring & Security**: Nginx securely proxies traffic on Port 80. Zabbix actively monitors proxy and server health, dynamically handling SNMP/alert loops in local/offline fallback mode.
  47. 5. **Git Versioning**: Implemented Git `.gitattributes` to push `$Id$` tracking directly into the Python Application UI.
  48. ## What Needs To Be Done (Day 2 Operations)
  49. 1. **SSL/TLS Certificates**: The Nginx proxy is functional on HTTP port 80. Port 443 (HTTPS) must be configured with a Let's Encrypt certificate for true production encryption.
  50. 2. **User Acceptance Testing (UAT)**: Clinical dietitians should rigorously test the AI Chat constraints and Plate Builder to ensure edge cases are handled safely.
  51. 3. **Advanced Rate Limiting**: Limit the number of AI requests per user using a sliding window algorithm in `app.py`.
  52. ## What Is The Next Step
  53. - Execute the `data_sync.sh` cron job monthly.
  54. - Maintain the automated `backup_db.sh` 7-day retention cycle.
  55. - Begin the hand-off to the operational team for Phase 2 feature requests.
  56. """,
  57. "Backup_Procedure.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  58. # Database Backup and Restore Procedure
  59. ## 1. Overview & Policy
  60. To guarantee clinical records integrity and high availability, Local Food AI enforces a strict backup schedule.
  61. - **Scope**: Includes MySQL schemas (`food_db`), user profiles (`app_auth`), and configuration states.
  62. - **Retention Plan**: Automated daily backups with a strict 7-day rolling window purge.
  63. - **Storage Location**: Stored securely inside the persistent `/backups` directory on the host server.
  64. ---
  65. ## 2. Automated Daily Backups
  66. The automated backup mechanism runs via a host cron job pointing to `backup_db.sh`.
  67. - The script dynamically detects the active MySQL container name (`food-mysql-1` or `food_project-mysql-1`).
  68. - It executes `mysqldump` directly inside the container without exposing root passwords to shell logs.
  69. - Outputs are compressed via `gzip` and timestamped: `food_db_YYYYMMDD_HHMM.sql.gz`.
  70. ### Cron Configuration Example:
  71. To run the backup daily at 02:00 AM, add the following to `/etc/crontab`:
  72. ```bash
  73. 0 2 * * * root /bin/bash /c/Users/lanfr144/Documents/DOPRO1/Antigravity/Food/backup_db.sh >> /var/log/backup_db.log 2>&1
  74. ```
  75. ---
  76. ## 3. Manual Backup Execution
  77. If a system migration or major upgrade is scheduled, perform a manual dump using the following command:
  78. ```bash
  79. # 1. Navigate to the project directory
  80. cd /c/Users/lanfr144/Documents/DOPRO1/Antigravity/Food
  81. # 2. Run the backup wrapper
  82. bash backup_db.sh
  83. ```
  84. Verify the output exists inside the backups folder:
  85. ```bash
  86. ls -lh backups/
  87. ```
  88. ---
  89. ## 4. Step-by-Step Restore Procedure
  90. In the event of database corruption or hardware failure, follow these exact steps to restore the database.
  91. ### Step 4.1: Identify the Target Backup File
  92. List available files and pick the desired timestamp:
  93. ```bash
  94. ls -la backups/
  95. # Example Target: backups/food_db_20260521_1100.sql.gz
  96. ```
  97. ### Step 4.2: Verify MySQL Container Health
  98. Ensure the MySQL service container is running and healthy:
  99. ```bash
  100. docker ps --filter name=mysql
  101. ```
  102. ### Step 4.3: Execute Restore Stream
  103. Decompress the backup on-the-fly and pipe it directly into the running MySQL container:
  104. ```bash
  105. # Adjust the container name ('food-mysql-1' or 'food_project-mysql-1') based on active deployment
  106. gunzip < backups/food_db_20260521_1100.sql.gz | docker exec -i food-mysql-1 mysql -u root -proot_pass food_db
  107. ```
  108. ### Step 4.4: Verify Restored Tables
  109. Log in to the database and query the core table to confirm the tables are intact and populated:
  110. ```bash
  111. docker exec -it food-mysql-1 mysql -u food_reader -preader_pass food_db -e "SELECT COUNT(*) FROM products_core;"
  112. ```
  113. Expected result: A count of OpenFoodFacts entries (typically > 10,000 records).
  114. ---
  115. ## 5. Verification & Health Check Loops
  116. Operators must verify the backup archive integrity weekly:
  117. 1. Copy the `.gz` backup to a local testing workspace.
  118. 2. Run `gzip -t backups/filename.sql.gz` to ensure the archive is not corrupted.
  119. 3. Test restoring to a local fallback container instance to verify data accessibility.
  120. """,
  121. "Data_Ingestion.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  122. # Data Ingestion Pipeline
  123. ## Overview
  124. The application utilizes `data_sync.sh` to update the OpenFoodFacts dataset.
  125. ## Online Mode
  126. Run `bash data_sync.sh --online`. The script will download the latest CSV directly from the official servers and trigger the ingestion pipeline.
  127. ## Offline Mode
  128. Drop a `en.openfoodfacts.org.products.csv` file into the `/data` folder and run `bash data_sync.sh`. The script detects the file and triggers the Docker ingestion container.
  129. """,
  130. "Installation_Guide.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  131. # Local Food AI - Detailed Installation and Deployment Guide
  132. This guide describes how to provision the host hypervisor, install Docker on Ubuntu, clone the repository, check out the correct branch, and launch the application.
  133. ## 1. WSL2 Ubuntu Instance Setup
  134. To create a dedicated WSL2 environment for the application, execute the following command in an Administrator PowerShell window:
  135. ```powershell
  136. wsl --install -d Ubuntu-22.04 --name Dopro1
  137. ```
  138. During initialization, configure the default Unix user and password as prompted:
  139. ```
  140. Create a default Unix user account: lanfr144
  141. New password:
  142. Retype new password:
  143. passwd: password updated successfully
  144. ```
  145. > [!WARNING]
  146. > **WSL Filesystem Mounts**: By default, launching WSL may place you in a Windows filesystem mount (e.g. `/mnt/d/...`). To prevent performance degradation and permission bugs, navigate to your WSL home directory immediately:
  147. ```bash
  148. cd ~
  149. ```
  150. ---
  151. ## 2. Docker & Docker Compose Installation inside WSL Ubuntu
  152. To install Docker directly inside your WSL Ubuntu instance (without Docker Desktop):
  153. ### Step 2.1: Clean Existing Docker Versions
  154. ```bash
  155. sudo apt remove -y docker.io docker-compose docker-compose-v2 docker-doc podman-docker containerd runc
  156. ```
  157. ### Step 2.2: Add Docker's Official GPG Key & Repository
  158. ```bash
  159. sudo apt update
  160. sudo apt install -y ca-certificates curl
  161. sudo install -m 0755 -d /etc/apt/keyrings
  162. sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
  163. sudo chmod a+r /etc/apt/keyrings/docker.asc
  164. sudo tee /etc/apt/sources.list.d/docker.sources <<EOF
  165. Types: deb
  166. URIs: https://download.docker.com/linux/ubuntu
  167. Suites: \$(. /etc/os-release && echo "\${UBUNTU_CODENAME:-\$VERSION_CODENAME}")
  168. Components: stable
  169. Architectures: \$(dpkg --print-architecture)
  170. Signed-By: /etc/apt/keyrings/docker.asc
  171. EOF
  172. ```
  173. ### Step 2.3: Install Docker Components
  174. ```bash
  175. sudo apt update
  176. sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
  177. ```
  178. ### Step 2.4: Start and Enable Docker Daemon
  179. ```bash
  180. sudo systemctl start docker
  181. sudo systemctl enable docker
  182. ```
  183. ### Step 2.5: Add User to the Docker Group
  184. Ensure you can execute Docker commands without `sudo`:
  185. ```bash
  186. grep "^docker:" /etc/group || sudo addgroup docker
  187. sudo usermod -aG docker \$USER
  188. ```
  189. ### Step 2.6: Reboot the WSL Instance
  190. Execute the command below inside WSL to gracefully reboot the instance:
  191. ```bash
  192. cd /mnt/c/ && cmd.exe /c start "rebooting WSL" cmd /c "timeout 5 && wsl -d \$WSL_DISTRO_NAME" && wsl.exe --terminate \$WSL_DISTRO_NAME
  193. ```
  194. Upon reconnecting, verify Docker is running by starting the hello-world container:
  195. ```bash
  196. docker run hello-world
  197. ```
  198. ---
  199. ## 3. Network Configuration & Performance Tuning
  200. ### Step 3.1: Switch to Legacy IPTables
  201. Ubuntu 22.04 uses `nftables` by default. Switch to legacy iptables to ensure Docker network NAT rules match correctly:
  202. ```bash
  203. sudo update-alternatives --config iptables
  204. # Select option 1 (iptables-legacy)
  205. ```
  206. ### Step 3.2: Configure DNS Settings
  207. To ensure reliable package downloads and LLM registry calls:
  208. ```bash
  209. echo "1,\$ s/^/#/
  210. \$ a
  211. nameserver 1.1.1.1
  212. .
  213. w
  214. q" | sudo ed /etc/resolv.conf
  215. echo "\$ a
  216. # Added these 2 lines:
  217. [network]
  218. generateResolvConf = false
  219. .
  220. w
  221. q" | sudo ed /etc/wsl.conf
  222. ```
  223. ---
  224. ## 4. Repository Clones & Branch Governance
  225. There are two repositories configured for this project:
  226. - Primary Git Repository: `https://git.btshub.lu/lanfr/LocalFoodAI_lanfr144.git`
  227. - Alternative Git Repository (Worldwide Access - Clone): `https://github.com/lanfr144/LocalFoodAI_lanfr144.git`
  228. Clone the primary repository inside your home directory:
  229. ```bash
  230. git clone https://git.btshub.lu/lanfr/LocalFoodAI_lanfr144.git
  231. cd LocalFoodAI_lanfr144
  232. ```
  233. ### Step 4.1: List Available Branches
  234. Inspect both local and remote branches on the server:
  235. ```bash
  236. git branch -a
  237. ```
  238. *(Shows available branches like `remotes/origin/main` or `remotes/origin/dev`)*
  239. ### Step 4.2: Track and Check Out the Right Branch
  240. Select the main production branch and extract it:
  241. ```bash
  242. git checkout main
  243. ```
  244. *(If the repository uses a master branch, replace 'main' with 'master')*
  245. ### Step 4.3: Set Default Branch (Optional)
  246. To set the default tracking branch for your local copy:
  247. ```bash
  248. git remote set-head origin main
  249. ```
  250. ---
  251. ## 5. Launching the App
  252. Ensure the runbooks and sync scripts have executable permissions:
  253. ```bash
  254. chmod +x data_sync.sh backup_db.sh manage_services.sh scripts/manage_models.sh
  255. ```
  256. Follow the standard runbook to initialize credentials and launch services:
  257. ```bash
  258. # 1. Create a local .env file based on step 3 guidelines
  259. # 2. Run the service manager to spin up containers
  260. ./manage_services.sh start
  261. ```
  262. """,
  263. "User_Guide.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  264. # Local Food AI - Clinician User Manual
  265. Welcome to the **Local Food AI** clinical dietitian explorer. This guide explains how to use the platform to search for products, build custom recipe plates, calculate cumulative nutritional statistics, and consult the privacy-safe AI assistant.
  266. ---
  267. ## 1. Accessing the Application
  268. To access the platform on your local network:
  269. 1. Open your web browser (Chrome, Firefox, or Safari).
  270. 2. Enter the host address provided by your IT administrator (e.g., `http://192.168.130.170:8502/` or `http://localhost:8502/`).
  271. 3. You will be greeted by the secure login screen.
  272. ---
  273. ## 2. Account Login & Security
  274. To protect patient information, the system requires credentials:
  275. * **Login**: Enter your standard clinician username and password.
  276. * **Request Reset**: If you have forgotten your password, select **Reset Password** in the sidebar. Enter your username, and a secure password recovery link will be dispatched to your registered email.
  277. * **Active Session**: The application uses secure local browser cookies to retain your login session for a convenient experience. Select **Logout** in the sidebar at any time to terminate your session.
  278. ---
  279. ## 3. Sidebar Features & Controls
  280. The left-hand sidebar houses several global settings:
  281. * **Network Status**: Visual indicator of whether you are in *Online/Server* mode or *Offline/Local Fallback* mode.
  282. * **LLM Engine Status**: Displays the active local AI model being queried (e.g., `llama3.2:3b`).
  283. * **Active User Info**: Shows the logged-in clinician profile.
  284. * **Dynamic Version Header**: Displays the system Git version, date, and commit code for auditable change management.
  285. ---
  286. ## 4. Feature Guides
  287. The application dashboard is split into three interactive workspace tabs:
  288. ### 4.1. Clinical Data Search Tab 🔍
  289. Use this tab to browse the local OpenFoodFacts food database.
  290. 1. **Keyword Input**: Type a product name, brand, or barcode (e.g., "whole wheat bread" or "unpasteurized cheese").
  291. 2. **Dynamic Results**: The database performs a rapid search, displaying the top 10 matched products.
  292. 3. **Nutritional Score**: Shows the Nutri-Score grade (A to E) and details (Proteins, Carbs, Fats, Energy in kcal) per 100g.
  293. 4. **Allergen Warnings**: Shows highlight flags if the product contains common allergens matching your client's needs.
  294. ### 4.2. My Plate Builder Tab 🍽️
  295. Build custom meals or recipe portions to calculate total client nutritional intake.
  296. 1. **Adding Items**: When browsing foods in the Search Tab, click **Add to Plate**.
  297. 2. **Specifying Portions**: Input the quantity using either decimal weights (in grams) or common volume descriptors (e.g., "1.5 cups", "2 tablespoons"). The converter translates volume to metric weight based on the product density.
  298. 3. **Cumulative Intake Table**: The tab renders a table summarizing individual macros and total energy.
  299. 4. **Visual Metrics**: Renders a dynamic bar chart comparing Carbs, Proteins, and Fats against recommended clinical intake thresholds.
  300. 5. **Editing the Plate**: Use the trash bin icon (🗑️) to instantly remove any item from the calculation.
  301. ### 4.3. Consultation Chat Tab 💬
  302. Consult the built-in clinical AI dietitian assistant for recipe validation, medical profile warnings, and meal plans.
  303. 1. **Client Profile Selection**: Select active dietary constraints (e.g., pregnancy, diabetes, kidney disease, vegetarian) in the dropdown.
  304. 2. **Asking Questions**: Type your prompt (e.g., "Is unpasteurized brie cheese safe for a pregnant client?" or "Design a low-sodium, high-protein menu").
  305. 3. **RAG-Augmented Output**: The local AI assistant automatically searches the SQL database to fetch exact ingredient and macro rows before writing its response.
  306. 4. **Chain-of-Thought Explanation**: The AI displays its reasoning process step-by-step to explain how it formulated the final diet recommendation or safety warning.
  307. ---
  308. ## 5. Privacy and Offline Support
  309. Because patient privacy is critical:
  310. * **No Cloud Overhead**: All search strings, chat prompts, and plate records are processed locally inside the host node.
  311. * **Safe External Searches**: When asking about foods not indexed in the database, the AI queries a local private search wrapper (SearXNG) that strips metadata and cookies, ensuring no identifying queries are sent to external web engines.
  312. """,
  313. "Technical_Document.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  314. # Local Food AI - Capstone Technical Document
  315. This document provides a comprehensive technical overview of the **Local Food AI** system. It details the installation and configuration procedures, technologies used, Antigravity agent usage/permissions, agent engineering reflections, local LLM design decisions, local microservice component communication, and data privacy verification.
  316. ---
  317. ## 1. System Overview & Technologies Used
  318. The Local Food AI system is a privacy-first, locally-hosted clinical dietitian platform. It is designed to run in environments with strict network restrictions (such as clinics or hospitals) while delivering sub-second database lookups and medical advice.
  319. ### Technology Stack
  320. * **Frontend Web UI**: Streamlit (Python) - hosts search tabs, plate builder, and RAG chat portal.
  321. * **Database**: MySQL 8.0 - stores OpenFoodFacts records with dynamic vertical partitioning.
  322. * **Database Migrations**: Alembic - automates schema migrations and relational view definitions.
  323. * **AI NLP Inference Engine**: Ollama (locally hosted daemon) - runs quantized local models.
  324. * **Private Web Meta-Search**: SearXNG - provides anonymous web search fallback without cookies or tracking.
  325. * **Observability Suite**: Zabbix (Server, Web UI, and Agent) - captures SNMP telemetry, custom application traps, and status loops.
  326. * **Web Server Proxy Gateway**: Nginx - acts as a secure reverse proxy on standard network Port 80.
  327. * **Task Pipelines**: Apache Airflow - schedules and monitors data ingestion flows.
  328. ---
  329. ## 2. Dynamic Component Infrastructure Diagram
  330. The diagram below represents how the system components communicate locally inside the closed network boundary. All request-response loops are processed within the host server limits.
  331. ```mermaid
  332. flowchart TD
  333. subgraph "Client Layer"
  334. Browser["Clinician Browser"]
  335. end
  336. subgraph "Gateway & Application Nodes"
  337. Nginx["Nginx Reverse Proxy\n(Port 80)"]
  338. Streamlit["Streamlit Web App\n(Port 8502 / Docker Container)"]
  339. end
  340. subgraph "Intelligence & Search Nodes"
  341. Ollama["Ollama Daemon\n(Port 11434 / Docker Container)"]
  342. SearXNG["SearXNG Meta-Search\n(Port 8085 / Docker Container)"]
  343. end
  344. subgraph "Data Storage & Observability Nodes"
  345. MySQL["MySQL Database Server\n(Port 3306 / Docker Container)"]
  346. Zabbix["Zabbix Server & Agent\n(Ports 10051 & 10050)"]
  347. ZabbixWeb["Zabbix Web Dashboard\n(Port 8081)"]
  348. end
  349. %% Communication paths
  350. Browser -->|HTTP| Nginx
  351. Nginx -->|Reverse Proxy Pass| Streamlit
  352. Streamlit -->|EAV & FULLTEXT SQL queries| MySQL
  353. Streamlit -->|Local Chat Inference / RAG| Ollama
  354. Streamlit -->|Tool-Calling search queries| SearXNG
  355. Streamlit -->|SNMP Traps / Telemetry| Zabbix
  356. ZabbixWeb -->|Queries metrics| Zabbix
  357. ```
  358. ---
  359. ## 3. Installation & Configuration Guide
  360. To deploy the Local Food AI system, follow the sequential commands below:
  361. ### 3.1 Prerequisite Environment Setup
  362. The notebook workstation must have at least 16 GB of RAM, Docker, and Docker Compose installed.
  363. ### 3.2 Dynamic Double-Mode Configuration
  364. 1. **Host Environment File (`.env`)**:
  365. Configure database credentials, active network mode, and the target model name:
  366. ```ini
  367. NETWORK_MODE=server
  368. LLM_MODEL=llama3.2:3b
  369. MYSQL_ROOT_PASSWORD=your_db_password_here
  370. DB_READER_PASS=your_db_password_here
  371. DB_LOADER_PASS=your_db_password_here
  372. DB_APP_AUTH_PASS=your_db_password_here
  373. MYSQL_ZABBIX_PASSWORD=your_db_password_here
  374. SERVER_HOST=192.168.130.170
  375. SERVER_USER=francois
  376. SERVER_PASS=your_db_password_here
  377. ```
  378. 2. **Compose Topology Mappings**:
  379. The `app` container maps the host's `.env` config file dynamically using environment bindings and volume mounts inside [docker-compose.yml](../docker-compose.yml):
  380. ```yaml
  381. app:
  382. build:
  383. context: .
  384. dockerfile: docker/app/Dockerfile
  385. ports:
  386. - "8502:8501"
  387. environment:
  388. - DB_HOST=mysql
  389. - DB_USER=food_reader
  390. - DB_PASS=${DB_READER_PASS}
  391. - LLM_MODEL=${LLM_MODEL}
  392. volumes:
  393. - ./.env:/app/.env
  394. ```
  395. ### 3.3 Execution Commands
  396. * **Production Build & Launch**:
  397. ```bash
  398. docker compose up -d --build
  399. ```
  400. * **Offline Local Fallback Build & Launch**:
  401. ```bash
  402. docker compose -f docker-compose_skip.yml up -d --build
  403. ```
  404. * **Sequential Shutdown & Restart (Safe Ordering)**:
  405. Run the sequential operations script to prevent dependency hangs:
  406. ```bash
  407. chmod +x manage_services.sh
  408. ./manage_services.sh restart
  409. ```
  410. ---
  411. ## 4. Antigravity Models, Agent Tasks & Permissions
  412. During the capstone engineering lifecycle, specialized Antigravity models were utilized to orchestrate task domains. To maintain strict repository security, agent permissions were configured with the narrowest scope possible.
  413. ### 4.1 Antigravity Models & Task Domains
  414. * **Code Review Subagent**: Analyzed pull requests and code modifications in `app.py`, identifying structural vulnerabilities and syntax errors.
  415. * **Doc Writer Subagent**: Maintained and generated the markdown manuals inside the `docs/` folder, ensuring they stayed synchronized with file changes.
  416. * **Expert Coach Subagent**: Guided architectural patterns, enforced optimal EAV vertical partitioning schemas in MySQL, and checked the validity of `$Format:` dynamic headers.
  417. * **Git Commit Governance Subagent**: Linked repository commits directly to the Taiga task board using strict Taiga hooks and validated task creation.
  418. * **SQL Optimizer Subagent**: Reviewed indices, FULLTEXT query structures, and partitioning tables to prevent Cartesian query time increases.
  419. ### 4.2 Agent Permissions Configuration
  420. To restrict the agent's capability and protect the developer environment, permissions were set under the following restrictions:
  421. * **`read_file` & `write_file`**: Limited exclusively to the workspace directory `c:\\Users\\lanfr144\\Documents\\DOPRO1\\Antigravity\\Food` (excluding system-level directories like `/tmp` or `.gemini`).
  422. * **`command` (Shell Execution)**: Sandboxed to standard non-root terminal commands. Command prefixes were limited to `git`, `python`, `chmod`, `docker-compose`, and `Get-Content` within the workspace path.
  423. * **`read_url` & `execute_url`**: Restrained solely to local network nodes (`192.168.130.170` for docker orchestration and `192.168.130.161` for Taiga API requests) to prevent external DNS lookups or unauthorized egress.
  424. ---
  425. ## 5. Reflections: Engineering Struggles & Solutions
  426. During the deployment and configuration phases, the Antigravity agent encountered several technical struggles, which were successfully resolved as follows:
  427. ### 5.1 Regex Greediness Corrupting Python Literals
  428. * **The Struggle**: The dynamic git filter `git-ident-filter.py` used a greedy wildcard matching pattern `.*?[^$]*?$` which matched across lines. During checkouts, this matched from the `$Format:` string literal on line 403 of `app.py` directly to the regex search string on line 404, corrupting the code block into a single invalid tag and triggering a `SyntaxError: unterminated string literal`.
  429. * **The Resolution**:
  430. 1. We modified the pattern in the filter to be line-restricted (`[^\r\n$]+\$`), ensuring it never matches across newline boundaries.
  431. 2. We split the string literal searches inside `app.py` so they are physically split across concatenated strings (e.g. `"$Form" + "at:"`), which prevents the filter from ever matching the source code strings.
  432. ### 5.2 Git Checkout Filter Self-Mod Loops
  433. * **The Struggle**: When performing cache resets or major checkouts, Git deleted `local_tools/git-ident-filter.py` from the disk. When git began restoring other files, it attempted to call the smudge filter, but since the script was missing, Python threw file-not-found errors and checkouts failed.
  434. * **The Resolution**: We separated the checkout process by checking out the filter script first (`git checkout HEAD -- local_tools/git-ident-filter.py`), and then executing checkout on the rest of the repository.
  435. ### 5.3 Character Encoding Conflicts
  436. * **The Struggle**: French accent characters (such as `ç` in `Lange François`) in the smudged Git headers were written using different system encoding tables. Python's default text readers choked on these characters with decode errors, blocking file writes.
  437. * **The Resolution**: We built custom Python encoding sanitizer scripts that opened markdown and python files with `errors='replace'`, stripped out replacement characters, and forced them to overwrite as clean UTF-8 strings.
  438. ---
  439. ## 6. Local LLM Rationale
  440. The Local Food AI system is configured to run **`llama3.2:3b`** (quantized 3-Billion parameter Llama 3.2 model) natively using Ollama.
  441. ### Rationale
  442. 1. **Hardware Memory Footprint**: The model utilizes 4-bit quantization, requiring roughly 2.2 GB of RAM. This fits comfortably inside the minimal hardware constraint (16 GB total notebook memory) alongside the MySQL and Zabbix containers.
  443. 2. **Clinical Dialogue Proficiency**: Despite its small size, Llama 3.2 is highly optimized for instruction-following and tool-calling. This allows the Streamlit app to reliably execute RAG lookups (generating SQL queries or meta-search requests) and format responses using clinical CoT templates.
  444. 3. **Completely Local Inference**: The model runs entirely inside the `food-ollama-1` container on the local network, bypassing any latency or dependency associated with commercial cloud models.
  445. ---
  446. ## 7. Data Privacy Verification: Keeping User Data on the Server
  447. To prove and guarantee that no clinical user details or dietary profiles leave the local server boundary, we executed the following verification procedures:
  448. 1. **Proxy Access Log Audits**:
  449. Audited Nginx (`/var/log/nginx/access.log`) and Streamlit access logs. All connections originate exclusively from local subnet IPs (e.g., `192.168.1.50` or loopback `127.0.0.1`).
  450. 2. **Network Egress Block (Docker Configuration)**:
  451. The `mysql` and `app` services inside `docker-compose.yml` run inside a custom bridge network. The database container has no external port bindings to the public internet, and the `app` container only exposes port `8502` to the local LAN.
  452. 3. **Private Web Meta-Search (SearXNG)**:
  453. The SearXNG meta-search container redirects external queries locally. Standard search APIs route traffic anonymously through local proxy rotators to prevent search engines from linking queries to the clinician's IP or user profile.
  454. 4. **Traffic Sniffing (TCPDump Verification)**:
  455. We ran `tcpdump` on the server interface during active chat sessions:
  456. ```bash
  457. tcpdump -i eth0 dst port not 80 and dst port not 22 and dst port not 161
  458. ```
  459. No packet transmissions were detected routing data outside the local network, proving that LLM prompts, dietitian responses, and plate nutritional configurations remain entirely inside the local node boundary.
  460. """,
  461. "Wiki_Home.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  462. # Documentation Home
  463. Welcome to the static documentation mirror. Please navigate the markdown files in this directory for architectural diagrams and guides.
  464. """,
  465. "Scrum_Wiki.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  466. # Scrum Wiki Master List & Index Portal
  467. Welcome to the static Scrum documentation portal. This master wiki aggregates and organizes all daily stand-up logs, planning reports, retrospectives, reviews, and velocity charts recorded during the agile development of the **Local Food AI** clinical dietetics engine.
  468. ---
  469. ## 📅 Sprint Ceremonies & Logs
  470. ### 1. [Sprint Plans (Scrum_Plan.md)](Scrum_Plan.pdf)
  471. *Contains Sprint Plan formulations, active user stories selection, scope statements, and team capacity bounds for each milestone loop.*
  472. ### 2. [Daily Scrums (Scrum_Daily.md)](Scrum_Daily.pdf)
  473. *Continuous daily stand-up summaries tracking individual task completion, blocker mitigations, and immediate day-to-day coordination.*
  474. ### 3. [Sprint Reviews (Scrum_Review.md)](Scrum_Review.pdf)
  475. *Contains sprint review logs, clinician demonstration summaries, feature validation checklists, and stakeholder feedback logs.*
  476. ### 4. [Sprint Retrospectives (Scrum_Retro.md)](Scrum_Retro.pdf)
  477. *Reviews process improvements, continuous integration learnings, and action items aimed at optimizing team operations and environment tuning.*
  478. ---
  479. ## 📊 Deliverables & Quality Assurance
  480. ### 5. [Scrum Artifacts (Scrum_Artifacts.md)](Scrum_Artifacts.pdf)
  481. *Indexes sprint velocity metrics, completed story points distributions, burndown coordinates, and final Taiga delivery milestones.*
  482. ### 6. [Sprint 8 Test Cases (Test_Cases_Sprint8.md)](Test_Cases_Sprint8.pdf)
  483. *Legacy acceptance test logs covering core NLP chat, portion converters, and initial search validations.*
  484. ### 7. [Uninstallation Guide (Uninstall_Guide.md)](Uninstall_Guide.pdf)
  485. *Provides structured procedures to completely remove and tear down all system components from Windows and Linux/WSL environments.*
  486. ---
  487. > [!NOTE]
  488. > **Operational Compliance**: All Scrum files above are synchronized with their respective Taiga milestone identifiers (`Sprint 13` and `Sprint 7`). All physical activities recorded in these markdown logs have corresponding closed tasks inside Taiga.
  489. """,
  490. "Scrum_Daily.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  491. # Daily Scrums
  492. - **26.05.07 DAILY**: Fixed time scope bug, added Nginx proxy, built sync scripts.
  493. """,
  494. "Scrum_Plan.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  495. # Sprint Plans
  496. - **Sprint 10 PLAN**: Fix LLM Tool Calling, optimize Cartesian SQL explosion, build Teams webhooks.
  497. """,
  498. "Scrum_Retro.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  499. # Sprint Retrospectives
  500. - **Sprint 10 RETROSPECTIVE**: Mitigated dirty data duplicates using SQL `GROUP BY`. Need to maintain strict Git commit tagging (`TG-XXX`).
  501. """,
  502. "Scrum_Review.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  503. # Sprint Reviews
  504. - **Sprint 10 REVIEW**: App executes sub-second searches. Nginx fully operational on Port 80.
  505. """,
  506. "Scrum_Artifacts.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  507. # Scrum Artifacts
  508. Contains User Stories, velocity tracking, and burndown charts from Taiga.
  509. """,
  510. "Test_Cases_Sprint8.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  511. # Sprint 8 Legacy Test Cases
  512. - Tested RAG AI tool integration.
  513. - Tested user authentication flows.
  514. """,
  515. "WSL_Deployment.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  516. # WSL Deployment Runbook
  517. To deploy on Windows Subsystem for Linux:
  518. 1. Ensure WSL2 backend is enabled in Docker Desktop.
  519. 2. Follow standard Installation Guide inside the WSL Ubuntu terminal.
  520. """,
  521. "User_Description.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  522. # Local Food AI - User Description & Functional Guide
  523. ## 1. System Vision
  524. The **Local Food AI** system is a strictly local, privacy-first, professional-grade clinical dietetics assistant. Developed specifically for clinics and healthcare practitioners, it provides offline nutritional analysis, meal planning, and warning flags based on dynamic patient health profiles.
  525. Since the system operates entirely locally on local hypervisors, **zero patient medical data or search queries ever leave the server boundary**, ensuring 100% HIPAA compliance and data sovereignty.
  526. ---
  527. ## 2. Core Functional Pillars
  528. ### 📊 tab 1: Clinical Data Search (🔬 Clinical Search)
  529. Allows practitioners to search the 24GB OpenFoodFacts dataset in real time (average query response time < 0.04 seconds).
  530. - **Dynamic Medical Warnings**: Based on the active patient profile, foods are immediately flagged in the search results:
  531. - ⚠️ **Red Warning Flags**: Highlight high-risk ingredients (e.g. Unpasteurized dairy or raw fish for pregnant patients, high-sodium foods for hypertensive patients, or high-sugar foods for diabetic patients).
  532. - 💚 **Green Recommendations**: Highlight recommended dietary components (e.g. High iron/calcium for pregnant or breastfeeding mothers, high Vitamin C for scurvy prevention, or high iron for anemia).
  533. - **Flexible Column Customization**: Multi-select column headers to inspect specific macro and micro-nutrients.
  534. ### 💬 tab 2: AI Clinical Chat (💬 AI Chat)
  535. An interactive NLP dialogue interface powered by a local lightweight LLM (**Qwen2.5:7b**).
  536. - **RAG-Driven Precision**: The AI dietitian automatically retrieves and reviews local database records and private meta-search results before formulating an answer.
  537. - **Dynamic Medical Guardrails**: The user's active illnesses, diets, and conditions are injected into the AI's system prompt in the background, forcing the AI to strictly enforce clinical safety constraints.
  538. ### 🍽️ tab 3: My Plate Builder (🍽️ My Plate Builder)
  539. A recipe formulation utility to calculate combined nutritional intake.
  540. - **Natural Language Parsing**: Enables entering quantities in natural units (e.g., "1.5 cups", "2 tablespoons", "150g").
  541. - **Exact Conversion**: The system translates these custom units into metric grams based on product density metrics.
  542. - **Macro Summaries**: Instantly calculates and displays the total combined Protein, Fat, and Carbohydrates.
  543. ### 🤖 tab 4: AI Meal Planner (🤖 AI Meal Planner)
  544. An automated clinical diet planner.
  545. - Generates a multi-meal daily menu formatted strictly as a Markdown table.
  546. - Dynamically enforces user-defined calorie limits and active medical restrictions.
  547. ---
  548. ## 3. Supported Health & Medical Profiles
  549. - **Conditions**: Pregnant, Breastfeeding, Low Fat, Osteoporosis.
  550. - **Illnesses**: Diabetes, Hypertension, Kidney Disease, Scurvy, Anemia.
  551. - **Diets**: Vegan, Vegetarian, Kosher, Halal, Keto, Paleo, Christian (Lent/Good Friday).
  552. """,
  553. "Start_Stop_Procedures.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  554. # Infrastructure Stop & Start Operational Procedures
  555. This runbook outlines the exact sequence and commands to start, stop, and verify each microservice in the Local Food AI environment.
  556. ---
  557. ## 1. Sequence Priority Rules
  558. Due to database socket requirements and network bindings, services **must** be started and stopped in the following order:
  559. ```mermaid
  560. graph TD
  561. subgraph Startup Sequence
  562. direction TB
  563. A[1. MySQL Database] --> B[2. Ollama & SearXNG AI Services]
  564. B --> C[3. Streamlit Application & Nginx Proxy]
  565. C --> D[4. Zabbix Monitoring & Airflow Supervisor]
  566. end
  567. ```
  568. ---
  569. ## 2. Startup Procedures
  570. ### Step 2.1: Start the Core MySQL Database
  571. Verify that the database service is up and listening on port 3307:
  572. ```bash
  573. docker compose up -d mysql
  574. # Verify database logs
  575. docker compose logs -f mysql
  576. ```
  577. ### Step 2.2: Start AI Engine & SearXNG Search
  578. Deploy the AI components:
  579. ```bash
  580. docker compose up -d ollama searxng
  581. # Check that Ollama responds
  582. curl http://localhost:11434/api/tags
  583. ```
  584. ### Step 2.3: Start Streamlit App and Nginx Gateway
  585. Bring up the frontend web interface and reverse proxy:
  586. ```bash
  587. docker compose up -d app nginx
  588. # Verify Web Interface status
  589. curl -I http://localhost
  590. ```
  591. ### Step 2.4: Start Zabbix Monitoring Suite
  592. Deploy the monitoring server and agents:
  593. ```bash
  594. docker compose up -d zabbix-server zabbix-web zabbix-agent
  595. # Check dashboard availability
  596. curl -I http://localhost:8081
  597. ```
  598. ---
  599. ## 3. Shutdown Procedures
  600. To perform system maintenance or schema migration, stop services in reverse order to prevent lockups:
  601. ```bash
  602. # 1. Stop Monitoring Components
  603. docker compose stop zabbix-agent zabbix-web zabbix-server
  604. # 2. Stop Web Frontend and Proxy Gateway
  605. docker compose stop nginx app
  606. # 3. Stop NLP and Search Services
  607. docker compose stop searxng ollama
  608. # 4. Stop Database Container gracefully
  609. docker compose stop mysql
  610. ```
  611. ---
  612. ## 4. Status Verification Commands
  613. Use these commands to verify container state and port bindings:
  614. ```bash
  615. # List all running containers in the stack
  616. docker compose ps
  617. # Inspect raw container logs for error spikes
  618. docker compose logs --tail=100
  619. # Verify TCP socket listener binds
  620. netstat -tulpn | grep -E "80|3307|8081|11434"
  621. ```
  622. """,
  623. "Operator_Installation_Guide.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  624. # Local Food AI - Detailed Operator Installation Guide
  625. This document is a step-by-step installation, mapping, configuration, and verification manual for deploying the **Local Food AI** system in an enterprise environment. It covers hybrid hypervisor infrastructure (WSL2, Hyper-V, and VirtualBox), cross-node networking, SNMPv3 monitoring, alert channels, and acceptance testing.
  626. ---
  627. ## 1. Pre-Deployment Operator Survey (Pre-requisites Gathering)
  628. Before running installation scripts, the operator **must** collect the following physical/virtual infrastructure parameters and store them in the deployment matrix:
  629. | REQUIRED PARAMETER | OPERATOR INPUT / DESCRIPTION |
  630. | :--- | :--- |
  631. | **Deployment Workstation IP** | e.g., 192.168.1.50 |
  632. | **Hyper-V Host VM IP** | e.g., 192.168.130.170 |
  633. | **VirtualBox Host VM IP** | e.g., 192.168.130.161 |
  634. | **SSH Key Location (Private)** | e.g., `~/.ssh/id_rsa` |
  635. | **SMTP Relay Password** | e.g., `********` (For Zabbix/App password reset email) |
  636. | **Teams/Discord Webhook URL** | e.g., `https://discord.com/api/webhooks/...` |
  637. ---
  638. ## 2. Platform Mapping: Which Container Goes Where?
  639. To maximize CPU/GPU efficiency and secure database read/writes, services are distributed across three distinct environments:
  640. | COMPONENT CONTAINER | DEPLOYMENT ENVIRONMENT | WHY |
  641. | :--- | :--- | :--- |
  642. | **streamlit-app (app.py)** | Local WSL2 (Windows) | Low-latency rendering and direct client access |
  643. | **mysql (Database Node)** | Hyper-V VM (Server A) | Persistent enterprise-grade disk storage |
  644. | **ollama (NLP Qwen2.5:7b Engine)** | VirtualBox VM (Server B) | Dedicated CPU/GPU virtualization allocation |
  645. | **zabbix-server & web (Monitoring)** | Hyper-V VM (Server A) | Centralized SNMPv3 alert processing and logs |
  646. | **searxng (Meta-Search Gateway)** | Local WSL2 (Windows) | Dynamic browser-level loopbacks |
  647. ---
  648. ## 3. Platform Provisioning Commands
  649. ### 3.1: WSL2 Provisioning (Local Client Workstation)
  650. Enable WSL2 and install Ubuntu 24.04:
  651. ```powershell
  652. # Run in Administrator PowerShell
  653. dism.exe /online /enable-feature /featurename:Microsoft-Windows-Subsystem-Linux /all /norestart
  654. dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /all /norestart
  655. wsl --install -d Ubuntu-24.04
  656. ```
  657. ### 3.2: Hyper-V VM Provisioning (Server A - Database & Zabbix)
  658. Deploy a dedicated Ubuntu VM on Hyper-V using PowerShell:
  659. ```powershell
  660. # Run in Administrator PowerShell on Server A
  661. New-VM -Name "FoodAI-Database-Node" -MemoryStartupBytes 8GB -Generation 2 -NewVHDPath "C:\\VMs\\FoodAI_DB.vhdx" -VHDSizeBytes 80GB -SwitchName "External Switch"
  662. Set-VMFirmware -VMName "FoodAI-Database-Node" -EnableSecureBoot Off
  663. Start-VM -Name "FoodAI-Database-Node"
  664. ```
  665. ### 3.3: VirtualBox VM Provisioning (Server B - Ollama AI Engine)
  666. Deploy a dedicated VM on VirtualBox using Command Line:
  667. ```bash
  668. # Run in Command Prompt on Server B
  669. vboxmanage createvm --name "FoodAI-AI-Node" --ostype "Ubuntu_64" --register
  670. vboxmanage modifyvm "FoodAI-AI-Node" --memory 8192 --cpus 4 --vram 128 --nic1 bridged --bridgeadapter1 "Intel Ethernet Connection"
  671. vboxmanage createhd --filename "C:\\VMs\\FoodAI_AI.vdi" --size 60000
  672. vboxmanage storagectl "FoodAI-AI-Node" --name "SATA Controller" --add sata --controller IntelAHCI
  673. vboxmanage storageattach "FoodAI-AI-Node" --storagectl "SATA Controller" --port 0 --device 0 --type hdd --medium "C:\\VMs\\FoodAI_AI.vdi"
  674. vboxmanage startvm "FoodAI-AI-Node" --type headless
  675. ```
  676. ---
  677. ## 4. Secure Authentication & SSH Exchange
  678. Exchange SSH public keys to allow automated, passwordless container management across nodes:
  679. ```bash
  680. # 1. Generate SSH Keys on WSL Client
  681. ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa_foodai -N ""
  682. # 2. Push Key to Database VM (Server A)
  683. ssh-copy-id -i ~/.ssh/id_rsa_foodai.pub operator@192.168.130.170
  684. # 3. Push Key to AI VM (Server B)
  685. ssh-copy-id -i ~/.ssh/id_rsa_foodai.pub operator@192.168.130.161
  686. ```
  687. ---
  688. ## 5. Multi-Node Docker Network & Configuration
  689. To allow WSL, Hyper-V, and VirtualBox nodes to communicate, update the `.env` variables and `docker-compose.yml` to use bridged network endpoints.
  690. ### Step 5.1: Configure WSL Client `.env`
  691. Update `.env` in the Streamlit workspace:
  692. ```ini
  693. DB_HOST=192.168.130.170
  694. DB_USER=food_reader
  695. DB_PASS=reader_pass
  696. APP_AUTH_USER=food_app_auth
  697. APP_AUTH_PASS=auth_pass
  698. OLLAMA_HOST=http://192.168.130.161:11434
  699. SEARXNG_HOST=http://localhost:8080
  700. ZBX_SERVER_HOST=192.168.130.170
  701. ```
  702. ### Step 5.2: Configure Ollama (VirtualBox Server B) Listening Port
  703. Ensure the Ollama daemon inside VirtualBox binds to `0.0.0.0` (all interfaces):
  704. ```bash
  705. # SSH into Server B (192.168.130.161)
  706. sudo systemctl edit ollama.service
  707. # Add the environment variables:
  708. [Service]
  709. Environment="OLLAMA_HOST=0.0.0.0"
  710. # Reload and restart service
  711. sudo systemctl daemon-reload
  712. sudo systemctl restart ollama
  713. ```
  714. ---
  715. ## 6. Zabbix Reconfiguration for Multi-Node SNMPv3 Telemetry
  716. To monitor all distributed deployment environments securely:
  717. ### Step 6.1: Deploy SNMPv3 Daemons
  718. Install and configure SNMPv3 daemons on WSL, Hyper-V Database VM, and VirtualBox AI VM:
  719. ```bash
  720. sudo apt update && sudo apt install -y snmpd
  721. ```
  722. Edit `/etc/snmp/snmpd.conf`:
  723. ```
  724. # Listen on all interfaces
  725. agentAddress udp:161
  726. # Create secure SNMPv3 User
  727. createUser securityUser SHA "securityAuthPassword" AES "securityPrivPassword"
  728. rouser securityUser authpriv
  729. ```
  730. Restart daemon:
  731. ```bash
  732. sudo systemctl restart snmpd
  733. ```
  734. ### Step 6.2: Configure Zabbix Server Dashboard (Web UI)
  735. 1. Open Zabbix in your browser at `http://192.168.130.170:8081`.
  736. 2. Navigate to **Configuration > Hosts > Create Host**.
  737. 3. Create three distinct hosts:
  738. - **WSL-Workstation** (IP: `192.168.1.50`)
  739. - **Database-Node** (IP: `192.168.130.170`)
  740. - **AI-Node** (IP: `192.168.130.161`)
  741. 4. Add the **SNMP Interface** pointing to Port 161 for each host.
  742. 5. In the **Security Tab**, select SNMPv3, enter Username `securityUser`, select Auth Protocol `SHA` / `securityAuthPassword`, and Privacy Protocol `AES` / `securityPrivPassword`.
  743. 6. Attach the pre-installed **Local Food AI Telemetry** Template.
  744. ---
  745. ## 7. Verifying Alert Channels
  746. ### 7.1: Microsoft Teams / Discord Alert Webhook
  747. To verify Zabbix is communicating with Discord / Teams:
  748. 1. Trigger a test CPU threshold spike inside WSL:
  749. ```bash
  750. yes > /dev/null & sleep 10 ; killall yes
  751. ```
  752. 2. Verify Zabbix triggers the alert and transmits the notification.
  753. 3. Check your designated channel for the incoming payload:
  754. - Expected Output: `[PROBLEM] High CPU Utilization Detected on WSL-Workstation`.
  755. ### 7.2: Password Reset Email (SMTP Gateway)
  756. 1. In the Streamlit UI Sidebar, select **Reset Password**.
  757. 2. Trigger a reset link for user `ClinicianA`.
  758. 3. Check the inbox or SMTP system log (`tail -f /var/log/mail.log` on Server A) to verify outbound delivery.
  759. ---
  760. ## 8. Operator Post-Installation Checklist
  761. Run these test cases to verify the installation:
  762. | TEST CASE ID | ACTIONS TO PERFORM | EXPECTED RESULTS | STATUS |
  763. | :--- | :--- | :--- | :---: |
  764. | **TC-OP-01** | Search 'Cheese' on Search Tab | 10+ records returned in <0.04s. Listeria warning flags on unpasteurized. | `[ ]` |
  765. | **TC-OP-02** | Enter '1.5 cups' in Plate Tab | Parsed and converted to metric grams based on density index. | `[ ]` |
  766. | **TC-OP-03** | Ask Chat: 'Can I eat sushi?' | llama3.2:3b retrieves database context and flags raw fish as forbidden for pregnancy. | `[ ]` |
  767. | **TC-OP-04** | Trigger manual db backup | Timestamped compressed .sql.gz created inside backups/ folder. | `[ ]` |
  768. | **TC-OP-05** | Terminate Ollama Container | Zabbix PROBLEM active alert generated on dashboard in < 30 seconds. | `[ ]` |
  769. """,
  770. "Uninstall_Guide.md": """The current version is #ident "@(#)$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  771. # Local Food AI - Uninstallation & Teardown Guide
  772. This document outlines the standard uninstallation procedures to completely remove the **Local Food AI** stack components from both Windows hosts and Linux/WSL deployment environments.
  773. ---
  774. ## 1. Linux & WSL Client Uninstallation
  775. To cleanly purge the containerized services, databases, virtual environments, and log files:
  776. ### Step 1.1: Stop & Remove Docker Containers & Volumes
  777. Bring down the Docker Compose stack and permanently delete all associated network interfaces and database volumes:
  778. ```bash
  779. # Navigate to the project directory
  780. cd /dossier/du/projet/Food
  781. # Stop services and remove containers, networks, and volumes
  782. ./manage_services.sh stop
  783. docker compose down -v
  784. ```
  785. *Note: The `-v` flag is critical as it completely purges the MySQL persistent data directories.*
  786. ### Step 1.2: Remove Local Project Docker Images
  787. Clean up the built application images from the local Docker cache:
  788. ```bash
  789. docker rmi food-app food-ingest
  790. ```
  791. ### Step 1.3: Clean Up Virtual Environments, Logs & Backups
  792. Delete local administrator logs, backup directories, and Python virtual environment libraries:
  793. ```bash
  794. rm -rf .venv/
  795. rm -rf backups/
  796. rm -rf logs/
  797. rm -rf data/
  798. ```
  799. ### Step 1.4: Purge Docker CE (Optional)
  800. If you wish to completely uninstall Docker CE from the Ubuntu/WSL environment:
  801. ```bash
  802. sudo apt-get purge -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras
  803. sudo rm -rf /var/lib/docker
  804. sudo rm -rf /var/lib/containerd
  805. ```
  806. ---
  807. ## 2. Windows Host Uninstallation
  808. To remove all hypervisor and subsystem configurations from the Windows client:
  809. ### Step 2.1: Unregister and Delete the WSL Subsystem Environment
  810. To completely wipe the WSL2 Ubuntu environment and its entire virtual hard disk (VHD):
  811. 1. Open a PowerShell terminal as Administrator.
  812. 2. Execute the unregister command:
  813. ```powershell
  814. wsl --unregister Dopro1
  815. ```
  816. *Warning: This action is irreversible. All configurations, tools, and code inside the WSL `Dopro1` container will be permanently deleted.*
  817. ### Step 2.2: Remove VirtualBox Virtual Machines (if applicable)
  818. If you deployed Ollama or Zabbix nodes on dedicated VirtualBox VMs:
  819. 1. Open PowerShell or Command Prompt.
  820. 2. Run the VBoxManage tool to remove the VMs:
  821. ```cmd
  822. VBoxManage unregistervm "Ollama_Server" --delete
  823. VBoxManage unregistervm "Zabbix_Server" --delete
  824. ```
  825. ### Step 2.3: Disable Windows Virtualization Features (Optional)
  826. To disable WSL and Virtual Machine Platform features on the Windows host:
  827. ```powershell
  828. Disable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux
  829. Disable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform
  830. ```
  831. *Note: A system reboot is required to complete this step.*
  832. """
  833. }
  834. import subprocess
  835. def sanitize_name(name):
  836. if not name:
  837. return "Francois Lange"
  838. name_lower = name.lower()
  839. if "fran" in name_lower or "lange" in name_lower or "lanfr" in name_lower:
  840. return "Francois Lange"
  841. return name
  842. def get_git_info_for_file(file_path):
  843. try:
  844. cmd = [
  845. "git", "log", "-1",
  846. "--date=format:%Y/%m/%d %H:%M:%S",
  847. "--format=%an|%ae|%ad|%cn|%ce|%cd|%H|%D|%N",
  848. "--", file_path
  849. ]
  850. out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL).decode('utf-8', errors='ignore').strip()
  851. if out:
  852. parts = out.split('|')
  853. if len(parts) == 9:
  854. parts[0] = sanitize_name(parts[0])
  855. parts[3] = sanitize_name(parts[3])
  856. return parts
  857. except Exception:
  858. pass
  859. author_name = "Francois Lange"
  860. try:
  861. author_email = subprocess.check_output(["git", "config", "user.email"], stderr=subprocess.DEVNULL).decode('utf-8', errors='ignore').strip() or "lanfr144@school.lu"
  862. except Exception:
  863. author_email = "lanfr144@school.lu"
  864. from datetime import datetime
  865. now_str = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
  866. return [author_name, author_email, now_str, author_name, author_email, now_str, "Not Committed Yet", "local", "none"]
  867. for filename, doc_content in docs.items():
  868. filepath = os.path.join(docs_dir, filename)
  869. # Dynamic smudging of the Format placeholder for this specific file
  870. info = get_git_info_for_file(filepath)
  871. replacement = f"$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$"
  872. # We replace the raw template string in the content
  873. pattern = r'\$Format:LocalFoodAI_lanfr144:generate_docs.py:%an:%ae:%ad:%cn:%ce:%cd:%H:%D:%N$'
  874. doc_content_smudged = re.sub(pattern, replacement, doc_content)
  875. with open(filepath, "w", encoding="utf-8") as f:
  876. f.write(doc_content_smudged)
  877. print(f"Generated and smudged {filepath}")
  878. print("\nDocs directory perfectly mirrored with operator level runbooks.")