[#1] Implement autonomous SQL/web RAG tools and default amino acid calculations in app.py

app.py

@@ -327,6 +327,57 @@ def search_nutrition_db(query: str, user_eav=None) -> str:
     finally:
         conn.close()
 
+def query_sql_readonly(sql: str) -> str:
+    """Executes a read-only SELECT query against the food_db MySQL database."""
+    sql_clean = sql.strip().strip(";").strip()
+    if not any(sql_clean.upper().startswith(kw) for kw in ["SELECT", "SHOW", "DESCRIBE", "EXPLAIN"]):
+        return "Error: Only read-only queries (SELECT, SHOW, DESCRIBE, EXPLAIN) are permitted."
+    
+    conn = get_db_connection('app_reader')
+    if not conn:
+        return "Error: Database connection failed."
+    try:
+        with conn.cursor() as cursor:
+            cursor.execute(sql)
+            results = cursor.fetchall()
+            if not results:
+                return "Query returned 0 rows."
+            
+            # Form custom markdown table
+            headers = list(results[0].keys())
+            lines = []
+            lines.append("| " + " | ".join(headers) + " |")
+            lines.append("| " + " | ".join(["---"] * len(headers)) + " |")
+            
+            # Cap at 10 results to stay within model context bounds
+            row_limit = 10
+            for row in results[:row_limit]:
+                vals = [str(row[h]).replace("\n", " ").replace("|", "\\|") for h in headers]
+                lines.append("| " + " | ".join(vals) + " |")
+            
+            truncated = f"\n*(Showing first {row_limit} of {len(results)} results)*" if len(results) > row_limit else ""
+            return "\n".join(lines) + truncated
+    except Exception as e:
+        return f"Database query error: {str(e)}"
+    finally:
+        conn.close()
+
+def query_web_searxng(query: str) -> str:
+    """Performs an anonymous local web search using the SearXNG search engine."""
+    try:
+        searxng_url = os.environ.get("SEARXNG_HOST", "http://searxng:8080")
+        resp = requests.get(f"{searxng_url}/search", params={'q': query, 'format': 'json'}, timeout=5)
+        if resp.status_code == 200:
+            results = resp.json().get('results', [])
+            if results:
+                snippets = []
+                for r in results[:4]:
+                    snippets.append(f"- **{r.get('title')}**: {r.get('content')} (Source: {r.get('url')})")
+                return "\n".join(snippets)
+        return "No web results found."
+    except Exception as e:
+        return f"Web search tool error: {str(e)}"
+
 db_search_tool_schema = {
     'type': 'function',
     'function': {
@@ -706,10 +757,12 @@ with tab_chat:
     if "messages" not in st.session_state:
         st.session_state["messages"] = [{"role": "assistant", "content": "How can I help you analyze the food data today?"}]
 
-    # Display chat history, filtering out TOOL_CALLS
+    # Display chat history, filtering out TOOL_CALLS and XML tags
     for msg in st.session_state.messages:
         if msg["role"] == "tool": continue
-        display_text = re.sub(r'\[TOOL_CALLS\]\s*\[.*?\]', '', msg["content"]).strip()
+        # Clean both old format [TOOL_CALLS] and new XML tags <sql_query>/<web_search>/<scratchpad>
+        display_text = re.sub(r'\[TOOL_CALLS\]\s*\[.*?\]', '', msg["content"], flags=re.DOTALL)
+        display_text = re.sub(r'<(sql_query|web_search|scratchpad)>.*?</\1>', '', display_text, flags=re.DOTALL).strip()
         if display_text:
             st.chat_message(msg["role"]).write(display_text)
 
@@ -720,38 +773,121 @@ with tab_chat:
         user_eav = get_eav_profile(st.session_state["authenticated_user"])
         profile_text = ", ".join([f"{p['name']}: {p['value']}" for p in user_eav]) if user_eav else "None"
         
-        db_context = search_nutrition_db(prompt, user_eav)
-        searxng_context = ""
-        
-        if "No database records found" in db_context:
-            try:
-                searxng_url = os.environ.get("SEARXNG_HOST", "http://searxng:8080")
-                resp = requests.get(f"{searxng_url}/search", params={'q': prompt, 'format': 'json'}, timeout=5)
-                if resp.status_code == 200:
-                    results = resp.json().get('results', [])
-                    if results:
-                        snippets = [r.get('content', '') for r in results[:3]]
-                        searxng_context = "Web Search Context: " + " | ".join(snippets)
-            except Exception as e:
-                pass
-                
-        sys_prompt = f"""You are a helpful medical data analyst AI. 
-        Health profile: {profile_text}. 
-        Act as a specialized clinical dietitian. Provide a direct answer. Use Chain of Thought reasoning, and skip pleasantries.
-        Local Database Context: {db_context}
-        {searxng_context}
-        """
+        sys_prompt = f"""You are a helpful, autonomous clinical dietitian and food analyst AI.
+You have access to a local MySQL database 'food_db' and a web search tool.
+You MUST determine dynamically when to query the database or search the web. Do not ask the user for permission.
+
+DATABASE SCHEMA (MySQL database 'food_db'):
+- Table 'products_core':
+  * 'code' (VARCHAR(255) PRIMARY KEY): barcode/unique key
+  * 'product_name' (LONGTEXT): name of product
+  * 'generic_name' (LONGTEXT): description
+  * 'brands' (LONGTEXT): brand
+  * 'ingredients_text' (LONGTEXT): text ingredient list
+- Table 'products_macros':
+  * 'code' (VARCHAR(255) PRIMARY KEY): joins products_core.code
+  * 'energy-kcal_100g' (DOUBLE): calories per 100g
+  * 'proteins_100g', 'fat_100g', 'carbohydrates_100g', 'sugars_100g', 'fiber_100g', 'sodium_100g', 'salt_100g', 'cholesterol_100g' (all DOUBLE)
+- Table 'products_vitamins':
+  * 'code' (VARCHAR(255) PRIMARY KEY): joins products_core.code
+  * `vitamin-a_100g`, `vitamin-b1_100g`, `vitamin-b2_100g`, `vitamin-pp_100g`, `vitamin-b6_100g`, `vitamin-b9_100g`, `vitamin-b12_100g`, `vitamin-c_100g`, `vitamin-d_100g`, `vitamin-e_100g`, `vitamin-k_100g` (all DOUBLE)
+- Table 'products_minerals':
+  * 'code' (VARCHAR(255) PRIMARY KEY): joins products_core.code
+  * 'calcium_100g', 'iron_100g', 'magnesium_100g', 'potassium_100g', 'zinc_100g' (all DOUBLE)
+- Table 'products_allergens':
+  * 'code' (VARCHAR(255) PRIMARY KEY): joins products_core.code
+  * 'allergens' (LONGTEXT): comma-separated allergens list
+
+TOOL TAGS (Generate these inside your response when you need details):
+1. To run a read-only SELECT query, write:
+   <sql_query>SELECT ...</sql_query>
+   Make sure to join tables on `code`. Optimize queries using LIKE. For example:
+   <sql_query>SELECT c.product_name, m.proteins_100g FROM products_core c JOIN products_macros m ON c.code = m.code WHERE c.product_name LIKE '%apple%' LIMIT 5;</sql_query>
+   
+2. To run a web search for information not in the database, write:
+   <web_search>query</web_search>
+
+CRITICAL CLINICAL RULES:
+- When asked about any food's nutritional profile, you MUST provide its typical amino acid profile (such as tryptophan, lysine, methionine, threonine, valine, leucine, isoleucine, phenylalanine, histidine, etc.) by default. If the database does not contain exact values, calculate/estimate them based on the food's protein content using standard clinical formulas and your clinical knowledge.
+- Health profile constraints: {profile_text}. If the user's health profile indicates restrictions (e.g. low sodium for hypertension, low sugar for diabetes, low protein for kidney issues), analyze the foods they query against these criteria and issue a clear clinical warning (⚠️ or 💚).
+- Do not micromanage or guess if you can query the database. Query the database first for exact product data whenever possible!
+
+INSTRUCTIONS:
+- First think in a <scratchpad> about what tools (if any) you need.
+- If you need database or web data, output the tool tag. The system will run it and give you the result.
+- Once you have the context or if you don't need any tool, output your final answer directly. Keep your reasoning brief and focus on details.
+"""
+
+        # We start the loop for autonomous tool calling
+        max_turns = 4
+        current_turn = 0
         
+        # Build messages list filtering system prompts
+        temp_messages = [{"role": "system", "content": sys_prompt}]
+        for m in st.session_state.messages:
+            if m["role"] == "system": continue
+            temp_messages.append({"role": m["role"], "content": m["content"]})
+            
         try:
-            temp_messages = [{"role": "system", "content": sys_prompt}] + [m for m in st.session_state.messages if m["role"] != "tool"]
+            full_assistant_reply = ""
             start_llm = time.time()
-            response_stream = ollama.chat(model=get_active_model(), messages=temp_messages, stream=True)
             
             with st.chat_message("assistant"):
-                ai_reply = st.write_stream(chunk['message']['content'] for chunk in response_stream)
+                placeholder = st.empty()
+                status_container = st.container()
+                
+                while current_turn < max_turns:
+                    current_turn += 1
+                    
+                    # Stream the response chunk by chunk
+                    response_stream = ollama.chat(model=get_active_model(), messages=temp_messages, stream=True)
+                    turn_reply = ""
+                    
+                    # Clean display text (strip scratchpad and tool tags while streaming)
+                    for chunk in response_stream:
+                        content = chunk['message']['content']
+                        turn_reply += content
+                        
+                        # Show stream output in real time
+                        display_text = re.sub(r'<(sql_query|web_search|scratchpad)>.*?</\1>', '', turn_reply, flags=re.DOTALL)
+                        display_text = re.sub(r'<(sql_query|web_search|scratchpad)>.*$', '', display_text, flags=re.DOTALL).strip()
+                        if display_text:
+                            placeholder.markdown(display_text)
+                            
+                    full_assistant_reply += "\n" + turn_reply
+                    
+                    # Check if there are tool tags in this turn's response
+                    sql_match = re.search(r'<sql_query>(.*?)</sql_query>', turn_reply, re.DOTALL)
+                    web_match = re.search(r'<web_search>(.*?)</web_search>', turn_reply, re.DOTALL)
+                    
+                    if sql_match:
+                        sql_query = sql_match.group(1).strip()
+                        with status_container.expander(f"🔍 Querying database: {sql_query}", expanded=True):
+                            db_results = query_sql_readonly(sql_query)
+                            st.write(db_results)
+                        
+                        # Add assistant output and tool result to context
+                        temp_messages.append({"role": "assistant", "content": turn_reply})
+                        temp_messages.append({"role": "user", "content": f"Database Results:\n{db_results}"})
+                    elif web_match:
+                        web_query = web_match.group(1).strip()
+                        with status_container.expander(f"🌐 Searching the web: {web_query}", expanded=True):
+                            web_results = query_web_searxng(web_query)
+                            st.write(web_results)
+                            
+                        temp_messages.append({"role": "assistant", "content": turn_reply})
+                        temp_messages.append({"role": "user", "content": f"Web Search Results:\n{web_results}"})
+                    else:
+                        # No tool calls generated. This is the final answer!
+                        break
+                
+                # Render the final cleaned reply to ensure it's displayed completely
+                final_clean_reply = re.sub(r'<(sql_query|web_search|scratchpad)>.*?</\1>', '', turn_reply, flags=re.DOTALL).strip()
+                placeholder.markdown(final_clean_reply)
                 st.caption(f"⏱️ AI response generated in {time.time() - start_llm:.2f} seconds")
+                
+            st.session_state.messages.append({"role": "assistant", "content": turn_reply})
             
-            st.session_state.messages.append({"role": "assistant", "content": ai_reply})
         except Exception as e: 
             error_msg = str(e)
             if "not found" in error_msg.lower() or "404" in error_msg.lower():